Ticket #6556: test-max-length.patch

File test-max-length.patch, 20.8 KB (added by zooko, 8 years ago)
  • twisted/protocols/basic.py

     
    452452                return self.lineLengthExceeded(line)
    453453            else:
    454454                self.lineReceived(line)
    455         if len(self._buffer) > self.MAX_LENGTH:
     455        if len(self._buffer) > (self.MAX_LENGTH + len(self.delimiter)):
    456456            return self.lineLengthExceeded(self._buffer)
    457457
    458458
     
    558558                        line, self._buffer = self._buffer.split(
    559559                            self.delimiter, 1)
    560560                    except ValueError:
    561                         if len(self._buffer) > self.MAX_LENGTH:
     561                        if len(self._buffer) > (self.MAX_LENGTH + len(self.delimiter)):
    562562                            line, self._buffer = self._buffer, b''
    563563                            return self.lineLengthExceeded(line)
    564564                        return
  • twisted/protocols/test/test_basic.py

     
    77
    88from __future__ import division, absolute_import
    99
     10import re
    1011import sys
    1112import struct
    1213from io import BytesIO
     
    5152
    5253
    5354
     55class BasicLineTester(basic.LineReceiver):
     56    """
     57    A line receiver that stores received lines in self.received.
     58    """
     59    def connectionMade(self):
     60        """
     61        Create/clean data received on connection.
     62        """
     63        self.received = []
     64
     65    def lineReceived(self, line):
     66        """
     67        Receive line and store it.
     68        """
     69        self.received.append(line)
     70   
     71
    5472class LineTester(basic.LineReceiver):
    5573    """
    5674    A line receiver that parses data received and make actions on some tokens.
     
    6482        you want to use the pause/rawpause functionalities.
    6583    """
    6684
    67     delimiter = b'\n'
    6885    MAX_LENGTH = 64
    6986
    70     def __init__(self, clock=None):
     87    def __init__(self, clock=None, delimiter = b'\n'):
    7188        """
    7289        If given, use a clock to make callLater calls.
    7390        """
     91        self.delimiter = delimiter
    7492        self.clock = clock
    7593
    7694
     
    154172    Test L{twisted.protocols.basic.LineReceiver}, using the C{LineTester}
    155173    wrapper.
    156174    """
    157     buffer = b'''\
    158 len 10
     175    buffer_template = b'''\
     176len 9+
    159177
    160 0123456789len 5
     1780123456789len 4+
    161179
    1621801234
    163 len 20
     181len 19+
    164182foo 123
    165183
    1661840123456789
     
    168186foo 5
    169187
    1701881234567890123456789012345678901234567890123456789012345678901234567890
    171 len 1
     189len 0+
    172190
    173191a'''
    174192
    175     output = [b'len 10', b'0123456789', b'len 5', b'1234\n',
    176               b'len 20', b'foo 123', b'0123456789\n012345678',
    177               b'len 0', b'foo 5', b'', b'67890', b'len 1', b'a']
     193    output_template = [b'len 9+', b'0123456789', b'len 4+', b'1234\n',
     194              b'len 19+', b'foo 123', b'0123456789\n012345678',
     195              b'len 0', b'foo 5', b'', b'67890', b'len 0+', b'a']
    178196
    179197    def test_buffer(self):
    180198        """
    181199        Test buffering for different packet size, checking received matches
    182200        expected data.
    183201        """
    184         for packet_size in range(1, 10):
    185             t = proto_helpers.StringIOWithoutClosing()
    186             a = LineTester()
    187             a.makeConnection(protocol.FileWrapper(t))
    188             for i in range(len(self.buffer) // packet_size + 1):
    189                 s = self.buffer[i * packet_size:(i + 1) * packet_size]
    190                 a.dataReceived(s)
    191             self.assertEqual(self.output, a.received)
     202        R=re.compile('(.*)([1-9][0-9]*)\+')
    192203
     204        # for delimsize in range(1, 10):
     205        for delimsize in range(2, 2):
     206            delimiter = '\r' * delimsize
     207            buffer = delimiter.join(self.buffer_template.split('\n'))
     208            output = []
     209            for tok_template in self.output_template:
     210                mo = R.match(tok_template)
     211                if mo:
     212                    n = int(mo.group(2))
     213                    tok = mo.group(1)+str(n+delimsize)
     214                    tok.replace('\n', delimiter)
     215                    output.append(tok)
     216                   
     217            # for packet_size in range(1, 10):
     218            for packet_size in range(1, 2):
     219                t = proto_helpers.StringIOWithoutClosing()
     220                a = LineTester(delimiter=delimiter)
     221                a.makeConnection(protocol.FileWrapper(t))
     222                for i in range(len(self.buffer) // packet_size + 1):
     223                    s = delimited_buffer[i * packet_size:(i + 1) * packet_size]
     224                    a.dataReceived(s)
     225                self.assertEqual(delimited_output, a.received)
    193226
     227
    194228    pauseBuf = b'twiddle1\ntwiddle2\npause\ntwiddle3\n'
    195229
    196230    pauseOutput1 = [b'twiddle1', b'twiddle2', b'pause']
     
    304338
    305339    def test_maximumLineLength(self):
    306340        """
    307         C{LineReceiver} disconnects the transport if it receives a line longer
    308         than its C{MAX_LENGTH}.
     341        C{LineReceiver} processes a line equal to its C{MAX_LENGTH}
     342        (not counting delimiter).
    309343        """
     344        proto = BasicLineTester()
     345        proto.MAX_LENGTH = 4
     346        t = proto_helpers.StringTransport()
     347        proto.makeConnection(protocol.FileWrapper(t))
     348        line = b'x' * proto.MAX_LENGTH
     349        proto.dataReceived(line + b'\r\n')
     350        self.assertFalse(t.disconnecting)
     351        self.assertEqual(line, proto.received and proto.received[0])
     352
     353
     354    def test_greaterThanMaximumLineLength(self):
     355        """
     356        C{LineReceiver} disconnects the transport if it receives a
     357        line longer than its C{MAX_LENGTH} + len(delimiter).
     358        """
    310359        proto = basic.LineReceiver()
    311360        transport = proto_helpers.StringTransport()
    312361        proto.makeConnection(transport)
    313         proto.dataReceived(b'x' * (proto.MAX_LENGTH + 1) + b'\r\nr')
     362        proto.dataReceived(b'x' * (proto.MAX_LENGTH + len(proto.delimiter) + 1) + b'\r\nr')
    314363        self.assertTrue(transport.disconnecting)
    315364
    316365
    317     def test_maximumLineLengthRemaining(self):
     366    def test_greaterThanMaximumLineLengthUnfinished(self):
    318367        """
    319         C{LineReceiver} disconnects the transport it if receives a non-finished
    320         line longer than its C{MAX_LENGTH}.
     368        C{LineReceiver} disconnects the transport it if receives a
     369        non-finished line longer than its C{MAX_LENGTH} +
     370        len(delimiter).
    321371        """
    322372        proto = basic.LineReceiver()
    323373        transport = proto_helpers.StringTransport()
    324374        proto.makeConnection(transport)
    325         proto.dataReceived(b'x' * (proto.MAX_LENGTH + 1))
     375        proto.dataReceived(b'x' * (proto.MAX_LENGTH + len(proto.delimiter) + 1))
    326376        self.assertTrue(transport.disconnecting)
    327377
    328378
     379    def test_notQuiteMaximumLineLengthUnfinished(self):
     380        """
     381        C{LineReceiver} doesn't disconnect the transport it if
     382        receives a non-finished line longer than its C{MAX_LENGTH} but
     383        shorter than its C{MAX_LENGTH} + len(delimiter). (When the
     384        first part that exceeds the max is the beginning of the
     385        delimiter.)
     386        """
     387        proto = basic.LineReceiver()
     388        # '\r\n' is the default, but we set it just to be explicit in this test.
     389        proto.delimiter = '\r\n'
     390        transport = proto_helpers.StringTransport()
     391        proto.makeConnection(transport)
     392        proto.dataReceived((b'x' * proto.MAX_LENGTH) + proto.delimiter[:len(proto.delimiter)-1])
     393        self.assertFalse(transport.disconnecting)
     394
     395
     396    # Actually, I don't mind if the implementation of LineReceiver
     397    # disconnects the transport at this point or not. If we allow it
     398    # to lazily not disconnect the transport yet, then this allows it
     399    # to manage this by merely counting bytes received without having
     400    # to inspect the bytes to see if they contain a partial
     401    # delimiter. That would allow it to be simpler and more
     402    # efficient. So I recommend not requiring this behavior from it,
     403    # and deleting this test.
     404    def disabled_test_notQuiteMaximumLineLengthUnfinishedNoPartialDelimiter(self):
     405        """
     406        C{LineReceiver} doesn't disconnect the transport it if
     407        receives a non-finished line longer than its C{MAX_LENGTH} but
     408        shorter than its C{MAX_LENGTH} + len(delimiter). (Even when
     409        the first part that exceeds the max is not the beginning of
     410        the delimiter.)
     411        """
     412        proto = basic.LineReceiver()
     413        # '\r\n' is the default, but we set it just to be explicit in this test.
     414        proto.delimiter = '\r\n'
     415        transport = proto_helpers.StringTransport()
     416        proto.makeConnection(transport)
     417        proto.dataReceived(b'x' * (proto.MAX_LENGTH + len(proto.delimiter) - 1))
     418        self.assertFalse(transport.disconnecting)
     419
     420
     421    def test_maximumLineLengthPartialDelimiter(self):
     422        """
     423        C{LineReceiver} doesn't disconnect the transport when it
     424        receives a finished line as long as its C{MAX_LENGTH}, when
     425        the second-to-last packet ended with a pattern that could have
     426        been -- and turns out to have been -- the start of a
     427        delimiter, and that packet causes the total input to exceed
     428        C{MAX_LENGTH} + len(delimiter).
     429        """
     430        proto = BasicLineTester()
     431        proto.MAX_LENGTH = 4
     432        t = proto_helpers.StringTransport()
     433        proto.makeConnection(t)
     434
     435        line = b'x' * (proto.MAX_LENGTH - 1)
     436        proto.dataReceived(line)
     437        proto.dataReceived(proto.delimiter[:-1])
     438        proto.dataReceived(proto.delimiter[-1:] + line)
     439        self.assertFalse(t.disconnecting)
     440        self.assertEqual(line, proto.received and proto.received[0])
     441
     442
     443    def test_moreThanMaximumLineLengthUnfinishedPartialDelimiter(self):
     444        """
     445        C{LineReceiver} disconnects the transport if it receives a
     446        non-finished line longer than its C{MAX_LENGTH} +
     447        len(delimiter), even when the second-to-last packet ended with
     448        a pattern that could have been the start of a delimiter.
     449        """
     450        class NoopReceiver(basic.LineReceiver):
     451            def lineReceived(self, line):
     452                pass
     453
     454        proto = NoopReceiver()
     455        proto.MAX_LENGTH = 4
     456        transport = proto_helpers.StringTransport()
     457        proto.makeConnection(transport)
     458
     459        line = b'x' * (proto.MAX_LENGTH - 1)
     460        proto.dataReceived(line)
     461        proto.dataReceived(proto.delimiter[0])
     462        proto.dataReceived(b'xx')
     463        self.assertTrue(transport.disconnecting)
     464
     465
     466    def test_lineLengthExceeded(self):
     467        """
     468        C{LineReceiver} calls C{lineLengthExceeded} with the entire
     469        remaining contents of its buffer.
     470        """
     471        caught_line = []
     472        class ExcessivelyLargeLineCatcher(basic.LineReceiver):
     473            def lineReceived(self, line):
     474                pass
     475            def lineLengthExceeded(self, line):
     476                caught_line.append(line)
     477
     478        proto = ExcessivelyLargeLineCatcher()
     479        proto.MAX_LENGTH=6
     480        transport = proto_helpers.StringTransport()
     481        proto.makeConnection(transport)
     482        excessive_input = b'x' * (proto.MAX_LENGTH * 2 + 2)
     483        proto.dataReceived(excessive_input)
     484        self.assertEqual(caught_line and caught_line[0], excessive_input)
     485        del caught_line[:]
     486
     487        excessive_input = b'x' * (proto.MAX_LENGTH * 2 + 2)
     488        proto.dataReceived(b'x'+proto.delimiter + excessive_input)
     489        self.assertEqual(caught_line[0], excessive_input)
     490        del caught_line[:]
     491
     492        excessive_input = b'x' * (proto.MAX_LENGTH * 2 + 2) + proto.delimiter + b'x' * (proto.MAX_LENGTH * 2 + 2)
     493        proto.dataReceived(excessive_input)
     494        self.assertEqual(caught_line[0], excessive_input)
     495        del caught_line[:]
     496
     497        excessive_input = b'x' * (proto.MAX_LENGTH * 2 + 2) + proto.delimiter + b'x' * (proto.MAX_LENGTH * 2 + 2) + proto.delimiter
     498        proto.dataReceived(excessive_input)
     499        self.assertEqual(caught_line[0], excessive_input)
     500        del caught_line[:]
     501
     502    def test_notQuiteMaximumLineLengthUnfinished(self):
     503        """
     504        C{LineReceiver} doesn't disconnect the transport it if
     505        receives a non-finished line whose length, counting the
     506        delimiter, is longer than its C{MAX_LENGTH} but shorter than
     507        its C{MAX_LENGTH} + len(delimiter). (When the first part that
     508        exceeds the max is the beginning of the delimiter.)
     509        """
     510        proto = basic.LineReceiver()
     511        # '\r\n' is the default, but we set it just to be explicit in this test.
     512        proto.delimiter = '\r\n'
     513        transport = proto_helpers.StringTransport()
     514        proto.makeConnection(transport)
     515        proto.dataReceived((b'x' * proto.MAX_LENGTH) + proto.delimiter[:len(proto.delimiter)-1])
     516        self.assertFalse(transport.disconnecting)
     517
     518
    329519    def test_rawDataError(self):
    330520        """
    331521        C{LineReceiver.dataReceived} forwards errors returned by
     
    381571        self.assertEqual(a.received, self.buffer.split(b'\n')[:-1])
    382572
    383573
    384     def test_lineTooLong(self):
     574    def test_maximumLineLength(self):
    385575        """
    386         Test sending a line too long: it should close the connection.
     576        C{LineOnlyReceiver} processes a line equal to its
     577        C{MAX_LENGTH} (not counting delimiter).
    387578        """
     579        proto = LineOnlyTester()
     580        proto.MAX_LENGTH = 4
    388581        t = proto_helpers.StringTransport()
    389         a = LineOnlyTester()
    390         a.makeConnection(t)
    391         res = a.dataReceived(b'x' * 200)
    392         self.assertIsInstance(res, error.ConnectionLost)
     582        proto.makeConnection(protocol.FileWrapper(t))
     583        line = b'x' * proto.MAX_LENGTH
     584        proto.dataReceived(line + b'\r\n')
     585        self.assertFalse(t.disconnecting)
     586        self.assertEqual(line, proto.received and proto.received[0])
    393587
    394588
     589    def test_greaterThanMaximumLineLength(self):
     590        """
     591        C{LineOnlyReceiver} disconnects the transport if it receives a
     592        line longer than its C{MAX_LENGTH} + len(delimiter).
     593        """
     594        proto = LineOnlyTester()
     595        transport = proto_helpers.StringTransport()
     596        proto.makeConnection(transport)
     597        proto.dataReceived(b'x' * (proto.MAX_LENGTH + len(proto.delimiter) + 1) + b'\r\nr')
     598        self.assertTrue(transport.disconnecting)
     599
     600
     601    def test_greaterThanMaximumLineLengthUnfinished(self):
     602        """
     603        C{LineOnlyReceiver} disconnects the transport it if receives a
     604        non-finished line longer than its C{MAX_LENGTH} +
     605        len(delimiter).
     606        """
     607        proto = LineOnlyTester()
     608        transport = proto_helpers.StringTransport()
     609        proto.makeConnection(transport)
     610        proto.dataReceived(b'x' * (proto.MAX_LENGTH + len(proto.delimiter) + 1))
     611        self.assertTrue(transport.disconnecting)
     612
     613
     614    def test_notQuiteMaximumLineLengthUnfinished(self):
     615        """
     616        C{LineOnlyReceiver} doesn't disconnect the transport it if
     617        receives a non-finished line longer than its C{MAX_LENGTH} but
     618        shorter than its C{MAX_LENGTH} + len(delimiter). (When the
     619        first part that exceeds the max is the beginning of the
     620        delimiter.)
     621        """
     622        proto = LineOnlyTester()
     623        # '\r\n' is the default, but we set it just to be explicit in this test.
     624        proto.delimiter = '\r\n'
     625        transport = proto_helpers.StringTransport()
     626        proto.makeConnection(transport)
     627        proto.dataReceived((b'x' * proto.MAX_LENGTH) + proto.delimiter[:len(proto.delimiter)-1])
     628        self.assertFalse(transport.disconnecting)
     629
     630
     631    # Actually, I don't mind if the implementation of LineOnlyReceiver
     632    # disconnects the transport at this point or not. If we allow it
     633    # to lazily not disconnect the transport yet, then this allows it
     634    # to manage this by merely counting bytes received without having
     635    # to inspect the bytes to see if they contain a partial
     636    # delimiter. That would allow it to be simpler and more
     637    # efficient. So I recommend not requiring this behavior from it,
     638    # and deleting this test.
     639    def disabled_test_notQuiteMaximumLineLengthUnfinishedNoPartialDelimiter(self):
     640        """
     641        C{LineOnlyReceiver} doesn't disconnect the transport it if
     642        receives a non-finished line longer than its C{MAX_LENGTH} but
     643        shorter than its C{MAX_LENGTH} + len(delimiter). (Even when
     644        the first part that exceeds the max is not the beginning of
     645        the delimiter.)
     646        """
     647        proto = LineOnlyTester()
     648        # '\r\n' is the default, but we set it just to be explicit in this test.
     649        proto.delimiter = '\r\n'
     650        transport = proto_helpers.StringTransport()
     651        proto.makeConnection(transport)
     652        proto.dataReceived(b'x' * (proto.MAX_LENGTH + len(proto.delimiter) - 1))
     653        self.assertFalse(transport.disconnecting)
     654
     655
     656    def test_maximumLineLengthPartialDelimiter(self):
     657        """
     658        C{LineOnlyReceiver} doesn't disconnect the transport when it
     659        receives a finished line as long as its C{MAX_LENGTH}, when
     660        the second-to-last packet ended with a pattern that could have
     661        been -- and turns out to have been -- the start of a
     662        delimiter, and that packet causes the total input to exceed
     663        C{MAX_LENGTH} + len(delimiter).
     664        """
     665        proto = LineOnlyTester()
     666        proto.MAX_LENGTH = 4
     667        t = proto_helpers.StringTransport()
     668        proto.makeConnection(t)
     669
     670        line = b'x' * (proto.MAX_LENGTH - 1)
     671        proto.dataReceived(line)
     672        proto.dataReceived(proto.delimiter[:-1])
     673        proto.dataReceived(proto.delimiter[-1:] + line)
     674        self.assertFalse(t.disconnecting)
     675        self.assertEqual(line, proto.received and proto.received[0])
     676
     677
     678    def test_moreThanMaximumLineLengthUnfinishedPartialDelimiter(self):
     679        """
     680        C{LineOnlyReceiver} disconnects the transport if it receives a
     681        non-finished line longer than its C{MAX_LENGTH} +
     682        len(delimiter), even when the second-to-last packet ended with
     683        a pattern that could have been the start of a delimiter.
     684        """
     685        proto = LineOnlyTester()
     686        proto.MAX_LENGTH = 4
     687        transport = proto_helpers.StringTransport()
     688        proto.makeConnection(transport)
     689
     690        line = b'x' * (proto.MAX_LENGTH - 1)
     691        proto.dataReceived(line)
     692        proto.dataReceived(proto.delimiter[0])
     693        proto.dataReceived(b'xx')
     694        self.assertTrue(transport.disconnecting)
     695
     696
     697    def test_lineLengthExceeded(self):
     698        """
     699        C{LineOnlyReceiver} calls C{lineLengthExceeded} with the
     700        entire remaining contents of its buffer.
     701        """
     702        caught_line = []
     703        class ExcessivelyLargeLineCatcher(LineOnlyTester):
     704            def lineReceived(self, line):
     705                pass
     706            def lineLengthExceeded(self, line):
     707                caught_line.append(line)
     708
     709        proto = ExcessivelyLargeLineCatcher()
     710        proto.MAX_LENGTH=6
     711        transport = proto_helpers.StringTransport()
     712        proto.makeConnection(transport)
     713        excessive_input = b'x' * (proto.MAX_LENGTH * 2 + 2)
     714        proto.dataReceived(excessive_input)
     715        self.assertEqual(caught_line and caught_line[0], excessive_input)
     716        del caught_line[:]
     717
     718        excessive_input = b'x' * (proto.MAX_LENGTH * 2 + 2)
     719        proto.dataReceived(b'x'+proto.delimiter + excessive_input)
     720        self.assertEqual(caught_line[0], excessive_input)
     721        del caught_line[:]
     722
     723        excessive_input = b'x' * (proto.MAX_LENGTH * 2 + 2) + proto.delimiter + b'x' * (proto.MAX_LENGTH * 2 + 2)
     724        proto.dataReceived(excessive_input)
     725        self.assertEqual(caught_line[0], excessive_input)
     726        del caught_line[:]
     727
     728        excessive_input = b'x' * (proto.MAX_LENGTH * 2 + 2) + proto.delimiter + b'x' * (proto.MAX_LENGTH * 2 + 2) + proto.delimiter
     729        proto.dataReceived(excessive_input)
     730        self.assertEqual(caught_line[0], excessive_input)
     731        del caught_line[:]
     732
     733
     734    def test_notQuiteMaximumLineLengthUnfinished(self):
     735        """
     736        C{LineOnlyReceiver} doesn't disconnect the transport it if
     737        receives a non-finished line whose length, counting the
     738        delimiter, is longer than its C{MAX_LENGTH} but shorter than
     739        its C{MAX_LENGTH} + len(delimiter). (When the first part that
     740        exceeds the max is the beginning of the delimiter.)
     741        """
     742        proto = LineOnlyTester()
     743        # '\r\n' is the default, but we set it just to be explicit in this test.
     744        proto.delimiter = '\r\n'
     745        transport = proto_helpers.StringTransport()
     746        proto.makeConnection(transport)
     747        proto.dataReceived((b'x' * proto.MAX_LENGTH) + proto.delimiter[:len(proto.delimiter)-1])
     748        self.assertFalse(transport.disconnecting)
     749
     750
    395751    def test_lineReceivedNotImplemented(self):
    396752        """
    397753        When L{LineOnlyReceiver.lineReceived} is not overridden in a subclass,