Ticket #6556: fix-max-length.patch

File fix-max-length.patch, 21.6 KB (added by zooko, 8 years ago)
  • twisted/protocols/test/test_basic.py

     
    77
    88from __future__ import division, absolute_import
    99
     10import re
    1011import sys
    1112import struct
    1213from io import BytesIO
     
    5152
    5253
    5354
     55class BasicLineTester(basic.LineReceiver):
     56    """
     57    A line receiver that stores received lines in self.received.
     58    """
     59    def connectionMade(self):
     60        """
     61        Create/clean data received on connection.
     62        """
     63        self.received = []
     64
     65    def lineReceived(self, line):
     66        """
     67        Receive line and store it.
     68        """
     69        self.received.append(line)
     70   
     71
    5472class LineTester(basic.LineReceiver):
    5573    """
    5674    A line receiver that parses data received and make actions on some tokens.
     
    6482        you want to use the pause/rawpause functionalities.
    6583    """
    6684
    67     delimiter = b'\n'
    6885    MAX_LENGTH = 64
    6986
    70     def __init__(self, clock=None):
     87    def __init__(self, clock=None, delimiter = b'\n'):
    7188        """
    7289        If given, use a clock to make callLater calls.
    7390        """
     91        self.delimiter = delimiter
    7492        self.clock = clock
    7593
    7694
     
    154172    Test L{twisted.protocols.basic.LineReceiver}, using the C{LineTester}
    155173    wrapper.
    156174    """
    157     buffer = b'''\
    158 len 10
     175    buffer_template = b'''\
     176len 9+
    159177
    160 0123456789len 5
     1780123456789len 4+
    161179
    1621801234
    163 len 20
     181len 19+
    164182foo 123
    165183
    1661840123456789
     
    168186foo 5
    169187
    1701881234567890123456789012345678901234567890123456789012345678901234567890
    171 len 1
     189len 0+
    172190
    173191a'''
    174192
    175     output = [b'len 10', b'0123456789', b'len 5', b'1234\n',
    176               b'len 20', b'foo 123', b'0123456789\n012345678',
    177               b'len 0', b'foo 5', b'', b'67890', b'len 1', b'a']
     193    output_template = [b'len 9+', b'0123456789', b'len 4+', b'1234\n',
     194              b'len 19+', b'foo 123', b'0123456789\n012345678',
     195              b'len 0', b'foo 5', b'', b'67890', b'len 0+', b'a']
    178196
    179197    def test_buffer(self):
    180198        """
    181199        Test buffering for different packet size, checking received matches
    182200        expected data.
    183201        """
    184         for packet_size in range(1, 10):
    185             t = proto_helpers.StringIOWithoutClosing()
    186             a = LineTester()
    187             a.makeConnection(protocol.FileWrapper(t))
    188             for i in range(len(self.buffer) // packet_size + 1):
    189                 s = self.buffer[i * packet_size:(i + 1) * packet_size]
    190                 a.dataReceived(s)
    191             self.assertEqual(self.output, a.received)
     202        R=re.compile('(.*)([1-9][0-9]*)\+')
    192203
     204        # for delimsize in range(1, 10):
     205        for delimsize in range(2, 2):
     206            delimiter = '\r' * delimsize
     207            buffer = delimiter.join(self.buffer_template.split('\n'))
     208            output = []
     209            for tok_template in self.output_template:
     210                mo = R.match(tok_template)
     211                if mo:
     212                    n = int(mo.group(2))
     213                    tok = mo.group(1)+str(n+delimsize)
     214                    tok.replace('\n', delimiter)
     215                    output.append(tok)
     216                   
     217            # for packet_size in range(1, 10):
     218            for packet_size in range(1, 2):
     219                t = proto_helpers.StringIOWithoutClosing()
     220                a = LineTester(delimiter=delimiter)
     221                a.makeConnection(protocol.FileWrapper(t))
     222                for i in range(len(self.buffer) // packet_size + 1):
     223                    s = delimited_buffer[i * packet_size:(i + 1) * packet_size]
     224                    a.dataReceived(s)
     225                self.assertEqual(delimited_output, a.received)
    193226
     227
    194228    pauseBuf = b'twiddle1\ntwiddle2\npause\ntwiddle3\n'
    195229
    196230    pauseOutput1 = [b'twiddle1', b'twiddle2', b'pause']
     
    304338
    305339    def test_maximumLineLength(self):
    306340        """
    307         C{LineReceiver} disconnects the transport if it receives a line longer
    308         than its C{MAX_LENGTH}.
     341        C{LineReceiver} processes a line equal to its C{MAX_LENGTH}
     342        (not counting delimiter).
    309343        """
     344        proto = BasicLineTester()
     345        proto.MAX_LENGTH = 4
     346        t = proto_helpers.StringTransport()
     347        proto.makeConnection(protocol.FileWrapper(t))
     348        line = b'x' * proto.MAX_LENGTH
     349        proto.dataReceived(line + b'\r\n')
     350        self.assertFalse(t.disconnecting)
     351        self.assertEqual(line, proto.received and proto.received[0])
     352
     353
     354    def test_greaterThanMaximumLineLength(self):
     355        """
     356        C{LineReceiver} disconnects the transport if it receives a
     357        line longer than its C{MAX_LENGTH} + len(delimiter).
     358        """
    310359        proto = basic.LineReceiver()
    311360        transport = proto_helpers.StringTransport()
    312361        proto.makeConnection(transport)
    313         proto.dataReceived(b'x' * (proto.MAX_LENGTH + 1) + b'\r\nr')
     362        proto.dataReceived(b'x' * (proto.MAX_LENGTH + len(proto.delimiter) + 1) + b'\r\nr')
    314363        self.assertTrue(transport.disconnecting)
    315364
    316365
    317     def test_maximumLineLengthRemaining(self):
     366    def test_greaterThanMaximumLineLengthUnfinished(self):
    318367        """
    319         C{LineReceiver} disconnects the transport it if receives a non-finished
    320         line longer than its C{MAX_LENGTH}.
     368        C{LineReceiver} disconnects the transport it if receives a
     369        non-finished line longer than its C{MAX_LENGTH} +
     370        len(delimiter).
    321371        """
    322372        proto = basic.LineReceiver()
    323373        transport = proto_helpers.StringTransport()
    324374        proto.makeConnection(transport)
    325         proto.dataReceived(b'x' * (proto.MAX_LENGTH + 1))
     375        proto.dataReceived(b'x' * (proto.MAX_LENGTH + len(proto.delimiter) + 1))
    326376        self.assertTrue(transport.disconnecting)
    327377
    328378
     379    def test_notQuiteMaximumLineLengthUnfinished(self):
     380        """
     381        C{LineReceiver} doesn't disconnect the transport it if
     382        receives a non-finished line longer than its C{MAX_LENGTH} but
     383        shorter than its C{MAX_LENGTH} + len(delimiter). (When the
     384        first part that exceeds the max is the beginning of the
     385        delimiter.)
     386        """
     387        proto = basic.LineReceiver()
     388        # '\r\n' is the default, but we set it just to be explicit in this test.
     389        proto.delimiter = '\r\n'
     390        transport = proto_helpers.StringTransport()
     391        proto.makeConnection(transport)
     392        proto.dataReceived((b'x' * proto.MAX_LENGTH) + proto.delimiter[:len(proto.delimiter)-1])
     393        self.assertFalse(transport.disconnecting)
     394
     395
     396    # Actually, I don't mind if the implementation of LineReceiver
     397    # disconnects the transport at this point or not. If we allow it
     398    # to lazily not disconnect the transport yet, then this allows it
     399    # to manage this by merely counting bytes received without having
     400    # to inspect the bytes to see if they contain a partial
     401    # delimiter. That would allow it to be simpler and more
     402    # efficient. So I recommend not requiring this behavior from it,
     403    # and deleting this test.
     404    def disabled_test_notQuiteMaximumLineLengthUnfinishedNoPartialDelimiter(self):
     405        """
     406        C{LineReceiver} doesn't disconnect the transport it if
     407        receives a non-finished line longer than its C{MAX_LENGTH} but
     408        shorter than its C{MAX_LENGTH} + len(delimiter). (Even when
     409        the first part that exceeds the max is not the beginning of
     410        the delimiter.)
     411        """
     412        proto = basic.LineReceiver()
     413        # '\r\n' is the default, but we set it just to be explicit in this test.
     414        proto.delimiter = '\r\n'
     415        transport = proto_helpers.StringTransport()
     416        proto.makeConnection(transport)
     417        proto.dataReceived(b'x' * (proto.MAX_LENGTH + len(proto.delimiter) - 1))
     418        self.assertFalse(transport.disconnecting)
     419
     420
     421    def test_maximumLineLengthPartialDelimiter(self):
     422        """
     423        C{LineReceiver} doesn't disconnect the transport when it
     424        receives a finished line as long as its C{MAX_LENGTH}, when
     425        the second-to-last packet ended with a pattern that could have
     426        been -- and turns out to have been -- the start of a
     427        delimiter, and that packet causes the total input to exceed
     428        C{MAX_LENGTH} + len(delimiter).
     429        """
     430        proto = BasicLineTester()
     431        proto.MAX_LENGTH = 4
     432        t = proto_helpers.StringTransport()
     433        proto.makeConnection(t)
     434
     435        line = b'x' * (proto.MAX_LENGTH - 1)
     436        proto.dataReceived(line)
     437        proto.dataReceived(proto.delimiter[:-1])
     438        proto.dataReceived(proto.delimiter[-1:] + line)
     439        self.assertFalse(t.disconnecting)
     440        self.assertEqual(line, proto.received and proto.received[0])
     441
     442
     443    def test_moreThanMaximumLineLengthUnfinishedPartialDelimiter(self):
     444        """
     445        C{LineReceiver} disconnects the transport if it receives a
     446        non-finished line longer than its C{MAX_LENGTH} +
     447        len(delimiter), even when the second-to-last packet ended with
     448        a pattern that could have been the start of a delimiter.
     449        """
     450        class NoopReceiver(basic.LineReceiver):
     451            def lineReceived(self, line):
     452                pass
     453
     454        proto = NoopReceiver()
     455        proto.MAX_LENGTH = 4
     456        transport = proto_helpers.StringTransport()
     457        proto.makeConnection(transport)
     458
     459        line = b'x' * (proto.MAX_LENGTH - 1)
     460        proto.dataReceived(line)
     461        proto.dataReceived(proto.delimiter[0])
     462        proto.dataReceived(b'xx')
     463        self.assertTrue(transport.disconnecting)
     464
     465
     466    def test_lineLengthExceeded(self):
     467        """
     468        C{LineReceiver} calls C{lineLengthExceeded} with the entire
     469        remaining contents of its buffer.
     470        """
     471        caught_line = []
     472        class ExcessivelyLargeLineCatcher(basic.LineReceiver):
     473            def lineReceived(self, line):
     474                pass
     475            def lineLengthExceeded(self, line):
     476                caught_line.append(line)
     477
     478        proto = ExcessivelyLargeLineCatcher()
     479        proto.MAX_LENGTH=6
     480        transport = proto_helpers.StringTransport()
     481        proto.makeConnection(transport)
     482        excessive_input = b'x' * (proto.MAX_LENGTH * 2 + 2)
     483        proto.dataReceived(excessive_input)
     484        self.assertEqual(caught_line and caught_line[0], excessive_input)
     485        del caught_line[:]
     486
     487        excessive_input = b'x' * (proto.MAX_LENGTH * 2 + 2)
     488        proto.dataReceived(b'x'+proto.delimiter + excessive_input)
     489        self.assertEqual(caught_line[0], excessive_input)
     490        del caught_line[:]
     491
     492        excessive_input = b'x' * (proto.MAX_LENGTH * 2 + 2) + proto.delimiter + b'x' * (proto.MAX_LENGTH * 2 + 2)
     493        proto.dataReceived(excessive_input)
     494        self.assertEqual(caught_line[0], excessive_input)
     495        del caught_line[:]
     496
     497        excessive_input = b'x' * (proto.MAX_LENGTH * 2 + 2) + proto.delimiter + b'x' * (proto.MAX_LENGTH * 2 + 2) + proto.delimiter
     498        proto.dataReceived(excessive_input)
     499        self.assertEqual(caught_line[0], excessive_input)
     500        del caught_line[:]
     501
     502    def test_notQuiteMaximumLineLengthUnfinished(self):
     503        """
     504        C{LineReceiver} doesn't disconnect the transport it if
     505        receives a non-finished line whose length, counting the
     506        delimiter, is longer than its C{MAX_LENGTH} but shorter than
     507        its C{MAX_LENGTH} + len(delimiter). (When the first part that
     508        exceeds the max is the beginning of the delimiter.)
     509        """
     510        proto = basic.LineReceiver()
     511        # '\r\n' is the default, but we set it just to be explicit in this test.
     512        proto.delimiter = '\r\n'
     513        transport = proto_helpers.StringTransport()
     514        proto.makeConnection(transport)
     515        proto.dataReceived((b'x' * proto.MAX_LENGTH) + proto.delimiter[:len(proto.delimiter)-1])
     516        self.assertFalse(transport.disconnecting)
     517
     518
     519    def test_notQuiteMaximumLineLengthUnfinished(self):
     520        """
     521        C{LineReceiver} doesn't disconnect the transport it if
     522        receives a non-finished line whose length, counting the
     523        delimiter, is longer than its C{MAX_LENGTH} but shorter than
     524        its C{MAX_LENGTH} + len(delimiter). (When the first part that
     525        exceeds the max is the beginning of the delimiter.)
     526        """
     527        proto = basic.LineReceiver()
     528        # '\r\n' is the default, but we set it just to be explicit in this test.
     529        proto.delimiter = '\r\n'
     530        transport = proto_helpers.StringTransport()
     531        proto.makeConnection(transport)
     532        proto.dataReceived((b'x' * proto.MAX_LENGTH) + proto.delimiter[:len(proto.delimiter)-1])
     533        self.assertFalse(transport.disconnecting)
     534
     535
    329536    def test_rawDataError(self):
    330537        """
    331538        C{LineReceiver.dataReceived} forwards errors returned by
     
    381588        self.assertEqual(a.received, self.buffer.split(b'\n')[:-1])
    382589
    383590
    384     def test_lineTooLong(self):
     591    def test_maximumLineLength(self):
    385592        """
    386         Test sending a line too long: it should close the connection.
     593        C{LineOnlyReceiver} processes a line equal to its
     594        C{MAX_LENGTH} (not counting delimiter).
    387595        """
     596        proto = LineOnlyTester()
     597        proto.MAX_LENGTH = 4
    388598        t = proto_helpers.StringTransport()
    389         a = LineOnlyTester()
    390         a.makeConnection(t)
    391         res = a.dataReceived(b'x' * 200)
    392         self.assertIsInstance(res, error.ConnectionLost)
     599        proto.makeConnection(protocol.FileWrapper(t))
     600        line = b'x' * proto.MAX_LENGTH
     601        proto.dataReceived(line + b'\r\n')
     602        self.assertFalse(t.disconnecting)
     603        self.assertEqual(line, proto.received and proto.received[0])
    393604
    394605
     606    def test_greaterThanMaximumLineLength(self):
     607        """
     608        C{LineOnlyReceiver} disconnects the transport if it receives a
     609        line longer than its C{MAX_LENGTH} + len(delimiter).
     610        """
     611        proto = LineOnlyTester()
     612        transport = proto_helpers.StringTransport()
     613        proto.makeConnection(transport)
     614        proto.dataReceived(b'x' * (proto.MAX_LENGTH + len(proto.delimiter) + 1) + b'\r\nr')
     615        self.assertTrue(transport.disconnecting)
     616
     617
     618    def test_greaterThanMaximumLineLengthUnfinished(self):
     619        """
     620        C{LineOnlyReceiver} disconnects the transport it if receives a
     621        non-finished line longer than its C{MAX_LENGTH} +
     622        len(delimiter).
     623        """
     624        proto = LineOnlyTester()
     625        transport = proto_helpers.StringTransport()
     626        proto.makeConnection(transport)
     627        proto.dataReceived(b'x' * (proto.MAX_LENGTH + len(proto.delimiter) + 1))
     628        self.assertTrue(transport.disconnecting)
     629
     630
     631    def test_notQuiteMaximumLineLengthUnfinished(self):
     632        """
     633        C{LineOnlyReceiver} doesn't disconnect the transport it if
     634        receives a non-finished line longer than its C{MAX_LENGTH} but
     635        shorter than its C{MAX_LENGTH} + len(delimiter). (When the
     636        first part that exceeds the max is the beginning of the
     637        delimiter.)
     638        """
     639        proto = LineOnlyTester()
     640        # '\r\n' is the default, but we set it just to be explicit in this test.
     641        proto.delimiter = '\r\n'
     642        transport = proto_helpers.StringTransport()
     643        proto.makeConnection(transport)
     644        proto.dataReceived((b'x' * proto.MAX_LENGTH) + proto.delimiter[:len(proto.delimiter)-1])
     645        self.assertFalse(transport.disconnecting)
     646
     647
     648    # Actually, I don't mind if the implementation of LineOnlyReceiver
     649    # disconnects the transport at this point or not. If we allow it
     650    # to lazily not disconnect the transport yet, then this allows it
     651    # to manage this by merely counting bytes received without having
     652    # to inspect the bytes to see if they contain a partial
     653    # delimiter. That would allow it to be simpler and more
     654    # efficient. So I recommend not requiring this behavior from it,
     655    # and deleting this test.
     656    def disabled_test_notQuiteMaximumLineLengthUnfinishedNoPartialDelimiter(self):
     657        """
     658        C{LineOnlyReceiver} doesn't disconnect the transport it if
     659        receives a non-finished line longer than its C{MAX_LENGTH} but
     660        shorter than its C{MAX_LENGTH} + len(delimiter). (Even when
     661        the first part that exceeds the max is not the beginning of
     662        the delimiter.)
     663        """
     664        proto = LineOnlyTester()
     665        # '\r\n' is the default, but we set it just to be explicit in this test.
     666        proto.delimiter = '\r\n'
     667        transport = proto_helpers.StringTransport()
     668        proto.makeConnection(transport)
     669        proto.dataReceived(b'x' * (proto.MAX_LENGTH + len(proto.delimiter) - 1))
     670        self.assertFalse(transport.disconnecting)
     671
     672
     673    def test_maximumLineLengthPartialDelimiter(self):
     674        """
     675        C{LineOnlyReceiver} doesn't disconnect the transport when it
     676        receives a finished line as long as its C{MAX_LENGTH}, when
     677        the second-to-last packet ended with a pattern that could have
     678        been -- and turns out to have been -- the start of a
     679        delimiter, and that packet causes the total input to exceed
     680        C{MAX_LENGTH} + len(delimiter).
     681        """
     682        proto = LineOnlyTester()
     683        proto.MAX_LENGTH = 4
     684        t = proto_helpers.StringTransport()
     685        proto.makeConnection(t)
     686
     687        line = b'x' * (proto.MAX_LENGTH - 1)
     688        proto.dataReceived(line)
     689        proto.dataReceived(proto.delimiter[:-1])
     690        proto.dataReceived(proto.delimiter[-1:] + line)
     691        self.assertFalse(t.disconnecting)
     692        self.assertEqual(line, proto.received and proto.received[0])
     693
     694
     695    def test_moreThanMaximumLineLengthUnfinishedPartialDelimiter(self):
     696        """
     697        C{LineOnlyReceiver} disconnects the transport if it receives a
     698        non-finished line longer than its C{MAX_LENGTH} +
     699        len(delimiter), even when the second-to-last packet ended with
     700        a pattern that could have been the start of a delimiter.
     701        """
     702        proto = LineOnlyTester()
     703        proto.MAX_LENGTH = 4
     704        transport = proto_helpers.StringTransport()
     705        proto.makeConnection(transport)
     706
     707        line = b'x' * (proto.MAX_LENGTH - 1)
     708        proto.dataReceived(line)
     709        proto.dataReceived(proto.delimiter[0])
     710        proto.dataReceived(b'xx')
     711        self.assertTrue(transport.disconnecting)
     712
     713
     714    def test_lineLengthExceeded(self):
     715        """
     716        C{LineOnlyReceiver} calls C{lineLengthExceeded} with the
     717        entire remaining contents of its buffer.
     718        """
     719        caught_line = []
     720        class ExcessivelyLargeLineCatcher(LineOnlyTester):
     721            def lineReceived(self, line):
     722                pass
     723            def lineLengthExceeded(self, line):
     724                caught_line.append(line)
     725
     726        proto = ExcessivelyLargeLineCatcher()
     727        proto.MAX_LENGTH=6
     728        transport = proto_helpers.StringTransport()
     729        proto.makeConnection(transport)
     730        excessive_input = b'x' * (proto.MAX_LENGTH * 2 + 2)
     731        proto.dataReceived(excessive_input)
     732        self.assertEqual(caught_line and caught_line[0], excessive_input)
     733        del caught_line[:]
     734
     735        excessive_input = b'x' * (proto.MAX_LENGTH * 2 + 2)
     736        proto.dataReceived(b'x'+proto.delimiter + excessive_input)
     737        self.assertEqual(caught_line[0], excessive_input)
     738        del caught_line[:]
     739
     740        excessive_input = b'x' * (proto.MAX_LENGTH * 2 + 2) + proto.delimiter + b'x' * (proto.MAX_LENGTH * 2 + 2)
     741        proto.dataReceived(excessive_input)
     742        self.assertEqual(caught_line[0], excessive_input)
     743        del caught_line[:]
     744
     745        excessive_input = b'x' * (proto.MAX_LENGTH * 2 + 2) + proto.delimiter + b'x' * (proto.MAX_LENGTH * 2 + 2) + proto.delimiter
     746        proto.dataReceived(excessive_input)
     747        self.assertEqual(caught_line[0], excessive_input)
     748        del caught_line[:]
     749
     750
     751    def test_notQuiteMaximumLineLengthUnfinished(self):
     752        """
     753        C{LineOnlyReceiver} doesn't disconnect the transport it if
     754        receives a non-finished line whose length, counting the
     755        delimiter, is longer than its C{MAX_LENGTH} but shorter than
     756        its C{MAX_LENGTH} + len(delimiter). (When the first part that
     757        exceeds the max is the beginning of the delimiter.)
     758        """
     759        proto = LineOnlyTester()
     760        # '\r\n' is the default, but we set it just to be explicit in this test.
     761        proto.delimiter = '\r\n'
     762        transport = proto_helpers.StringTransport()
     763        proto.makeConnection(transport)
     764        proto.dataReceived((b'x' * proto.MAX_LENGTH) + proto.delimiter[:len(proto.delimiter)-1])
     765        self.assertFalse(transport.disconnecting)
     766
     767
    395768    def test_lineReceivedNotImplemented(self):
    396769        """
    397770        When L{LineOnlyReceiver.lineReceived} is not overridden in a subclass,
  • twisted/protocols/basic.py

     
    452452                return self.lineLengthExceeded(line)
    453453            else:
    454454                self.lineReceived(line)
    455         if len(self._buffer) > self.MAX_LENGTH:
     455        if len(self._buffer) > (self.MAX_LENGTH + len(self.delimiter)):
    456456            return self.lineLengthExceeded(self._buffer)
    457457
    458458
     
    558558                        line, self._buffer = self._buffer.split(
    559559                            self.delimiter, 1)
    560560                    except ValueError:
    561                         if len(self._buffer) > self.MAX_LENGTH:
     561                        if len(self._buffer) > (self.MAX_LENGTH + len(self.delimiter)):
    562562                            line, self._buffer = self._buffer, b''
    563563                            return self.lineLengthExceeded(line)
    564564                        return