Ticket #3795: secure-chunked-3795-1.diff

File secure-chunked-3795-1.diff, 12.8 KB (added by ivank, 8 years ago)

fix _ChunkedTransferDecoder by limiting self._buffer for chunk-length and trailer, and completely ignoring chunk extension data.

  • twisted/web/http.py

    === modified file 'twisted/web/http.py' (properties changed: -x to +x)
     
    99
    1010Future Plans:
    1111 - HTTP client support will at some point be refactored to support HTTP/1.1.
    12  - Accept chunked data from clients in server.
    1312 - Other missing HTTP features from the RFC.
    1413
    1514Maintainer: Itamar Shtull-Trauring
     
    3635from twisted.python import log
    3736try: # try importing the fast, C version
    3837    from twisted.protocols._c_urlarg import unquote
     38    unquote # shut up pyflakes
    3939except ImportError:
    4040    from urllib import unquote
    4141
     
    13131313    @ivar finish: A flag indicating that the last chunk has been started.  When
    13141314        it finishes, the state will change to C{'finished'} and no more data
    13151315        will be accepted.
     1316
     1317    @ivar _maximumChunkSize: The maximum size (in bytes) of chunk to
     1318        support. The implementation MAY accept larger chunks.
    13161319    """
    13171320    state = 'chunk-length'
    13181321    finish = False
    13191322
     1323    # 2**64 in hex is 10000000000000000 and requires 17 bytes to represent
     1324    # in chunked encoding. This is used only to limit the amount of (non-data)
     1325    # bytes to be buffered.
     1326    _maximumChunkSize = 2**64
     1327
    13201328    def __init__(self, dataCallback, finishCallback):
    13211329        self.dataCallback = dataCallback
    13221330        self.finishCallback = finishCallback
    13231331        self._buffer = ''
     1332        self._maximumChunkLengthBytes = len(hex(self._maximumChunkSize)[2:])
    13241333
    13251334
    13261335    def dataReceived(self, data):
     
    13311340        data = self._buffer + data
    13321341        self._buffer = ''
    13331342        while data:
    1334             if self.state == 'chunk-length':
     1343            if self.state == 'ignore-extension':
     1344                if '\r' in data:
     1345                    if '\r\n' in data:
     1346                        self.state = 'chunk-length'
     1347                    else:
     1348                        self._buffer = self._savedPreExtensionChunkBytes
     1349                        if data[-1] == '\r':
     1350                            # The next byte to arrive could be a \n.
     1351                            # In that case, the 'ignore-extension' logic will run again
     1352                            # and switch to state 'chunk-length'.
     1353
     1354                            # The next byte to arrive could be NOT \n.
     1355                            # In that case, the stray \r in the chunk extension will be
     1356                            # ignored, and self._buffer will be reset back to
     1357                            # self._savedPreExtensionChunkBytes
     1358
     1359                            self._buffer += '\r'
     1360                        data = ''
     1361                else:
     1362                    self._buffer = self._savedPreExtensionChunkBytes
     1363                    data = ''
     1364            elif self.state == 'chunk-length':
    13351365                if '\r\n' in data:
    13361366                    line, rest = data.split('\r\n', 1)
    13371367                    parts = line.split(';')
    1338                     self.length = int(parts[0], 16)
    1339                     if self.length == 0:
     1368                    if len(parts[0].strip()) > self._maximumChunkLengthBytes:
     1369                        raise RuntimeError(
     1370                            "_ChunkedTransferDecoder.dataReceived got strange bytes for "
     1371                            "chunk size in parts %s" % (repr(parts),))
     1372                    try:
     1373                        length = int(parts[0], 16)
     1374                    except ValueError:
     1375                        raise RuntimeError(
     1376                            "_ChunkedTransferDecoder.dataReceived got invalid "
     1377                            "chunk size in parts %s" % (repr(parts),))
     1378                    self.length = length
     1379                    if self.length < 0:
     1380                        raise RuntimeError(
     1381                            "_ChunkedTransferDecoder.dataReceived got negative "
     1382                            "chunk size in parts %s" % (repr(parts),))
     1383                    elif self.length == 0:
    13401384                        self.state = 'trailer'
    13411385                        self.finish = True
    13421386                    else:
    13431387                        self.state = 'body'
    13441388                    data = rest
    13451389                else:
    1346                     self._buffer = data
    1347                     data = ''
     1390                    beforeSemicolon = data.split(';', 1)[0].strip()
     1391                    if len(beforeSemicolon) > self._maximumChunkLengthBytes:
     1392                        raise RuntimeError(
     1393                            "_ChunkedTransferDecoder.dataReceived"
     1394                            "got too many bytes for chunk-length.")
     1395                    if ';' in data:
     1396                        self._savedPreExtensionChunkBytes = beforeSemicolon
     1397                        self.state = 'ignore-extension'
     1398                        data = ''
     1399                    else:
     1400                        self._buffer = data
     1401                        data = ''
    13481402            elif self.state == 'trailer':
    13491403                if data.startswith('\r\n'):
    13501404                    data = data[2:]
    13511405                    if self.finish:
    13521406                        self.state = 'finished'
    1353                         self.finishCallback(data)
     1407                        self.finishCallback(data) # here, data is the terminal chunk
    13541408                        data = ''
    13551409                    else:
    13561410                        self.state = 'chunk-length'
    1357                 else:
     1411                elif data == '\r':
    13581412                    self._buffer = data
    13591413                    data = ''
     1414                else:
     1415                    raise RuntimeError(
     1416                        "_ChunkedTransferDecoder.dataReceived got bytes %s "
     1417                        "instead of the desired CRLF trailer." % (repr(data),))
    13601418            elif self.state == 'body':
    13611419                if len(data) >= self.length:
    13621420                    chunk, data = data[:self.length], data[self.length:]
  • twisted/web/test/test_http.py

    === modified file 'twisted/web/test/test_http.py' (properties changed: -x to +x)
     
    461461        self.assertEqual(L, ['abc', '12345', '0123456789'])
    462462
    463463
     464    def test_decodingBadTrailerFirstByte(self):
     465        """
     466        L{_ChunkedTransferDecoder.dataReceived} raises RuntimeError when
     467        it expects the first byte of the trailer but gets something else.
     468        """
     469        L = []
     470        p = http._ChunkedTransferDecoder(L.append, None)
     471        p.dataReceived('3\r\nabc\r\n4\r\n1234')
     472        self.assertRaises(RuntimeError, lambda: p.dataReceived('5'))
     473
     474
     475    def test_decodingButBadSecondByte(self):
     476        """
     477        L{_ChunkedTransferDecoder.dataReceived} raises RuntimeError when
     478        it expects the second byte of the trailer but gets something else.
     479        """
     480        L = []
     481        p = http._ChunkedTransferDecoder(L.append, None)
     482        p.dataReceived('3\r\nabc\r\n4\r\n1234')
     483        p.dataReceived('\r')
     484        self.assertRaises(RuntimeError, lambda: p.dataReceived('x'))
     485
     486
     487    def test_chunkLengthNegative(self):
     488        """
     489        L{_ChunkedTransferDecoder.dataReceived} raises RuntimeError
     490        when the chunk length string is negative.
     491        """
     492        L = []
     493        p = http._ChunkedTransferDecoder(L.append, None)
     494        self.assertRaises(RuntimeError, lambda: p.dataReceived('-3\r\nsomething'))
     495
     496
     497    def test_chunkLengthNegativeZero(self):
     498        """
     499        L{_ChunkedTransferDecoder.dataReceived} does not raise RuntimeError
     500        when the chunk length is "-0".
     501
     502        At this time, stricter RFC2616 validation would bring no benefits.
     503        """
     504        L = []
     505        finished = []
     506        p = http._ChunkedTransferDecoder(L.append, finished.append)
     507        p.dataReceived('1\r\nX\r\n-0\r\n\r\n')
     508        self.assertEqual(L, ['X'])
     509        self.assertEqual(finished, [''])
     510
     511
     512    def test_tooLongChunkLength(self):
     513        """
     514        L{_ChunkedTransferDecoder.dataReceived} raises RuntimeError
     515        when the chunk length string is too long.
     516        """
     517        L = []
     518        p = http._ChunkedTransferDecoder(L.append, None)
     519        max = p._maximumChunkLengthBytes
     520
     521        self.assertRaises(RuntimeError, lambda: p.dataReceived(
     522            ('9' * (max+1)) + '\r\n' + ('s' * (max+1))))
     523
     524
     525    def test_tooLongChunkLengthWithExtension(self):
     526        """
     527        L{_ChunkedTransferDecoder.dataReceived} immediately raises
     528        RuntimeError when the chunk length string is too long, even when
     529        it contains the beginning of a chunk extension.
     530        """
     531        L = []
     532        p = http._ChunkedTransferDecoder(L.append, None)
     533        max = p._maximumChunkLengthBytes
     534
     535        self.assertRaises(RuntimeError, lambda: p.dataReceived(
     536            (('9' * (max+1)) + ';')))
     537
     538
     539    def test_tooLongChunkLengthWithExtensionGoodMath(self):
     540        """
     541        L{_ChunkedTransferDecoder.dataReceived} doesn't include
     542        the length of the semicolon when determining the length
     543        of the chunked length string.
     544        """
     545        L = []
     546        p = http._ChunkedTransferDecoder(L.append, None)
     547        max = p._maximumChunkLengthBytes
     548
     549        p.dataReceived((('9' * (max)) + ';'))
     550
     551
     552    def test_chunkTooLongSeparate(self):
     553        """
     554        L{_ChunkedTransferDecoder.dataReceived} raises RuntimeError
     555        when the chunk length string is too long, even when the last byte
     556        is sent separately.
     557        """
     558        L = []
     559        p = http._ChunkedTransferDecoder(L.append, None)
     560        max = p._maximumChunkLengthBytes
     561
     562        p.dataReceived(('9' * max))
     563
     564        self.assertRaises(RuntimeError, lambda: p.dataReceived('9'))
     565
     566
     567    def test_chunkInvalidHex(self):
     568        """
     569        L{_ChunkedTransferDecoder.dataReceived} raises RuntimeError
     570        when the chunk is invalid hex.
     571        """
     572        L = []
     573        p = http._ChunkedTransferDecoder(L.append, None)
     574        self.assertRaises(RuntimeError, lambda: p.dataReceived(
     575            '9G\r\nsomething'))
     576
     577
     578    def test_chunkInvalidGarbage(self):
     579        """
     580        L{_ChunkedTransferDecoder.dataReceived} raises RuntimeError
     581        when the chunk is garbage.
     582        """
     583        L = []
     584        p = http._ChunkedTransferDecoder(L.append, None)
     585        self.assertRaises(RuntimeError, lambda: p.dataReceived(
     586            '#$\][,.\'!@#*&\r\nsomething'))
     587
     588
     589    def test_shortStrayCRInExtension(self):
     590        """
     591        L{_ChunkedTransferDecoder.dataReceived} ignores CR in chunk
     592        extensions.
     593        """
     594        L = []
     595        finished = []
     596        p = http._ChunkedTransferDecoder(L.append, finished.append)
     597        for s in '3\r\nabc\r\n5; extension=some\rthing\r\n12345\r\n0\r\n\r\n':
     598            p.dataReceived(s)
     599        self.assertEqual(L, ['a', 'b', 'c', '1', '2', '3', '4', '5'])
     600        self.assertEqual(finished, [''])
     601
     602
    464603    def test_short(self):
    465604        """
    466605        L{_ChunkedTransferDecoder.dataReceived} decodes chunks broken up and
     
    497636        self.assertEqual(L, ['abc'])
    498637
    499638
     639    def test_extensions(self):
     640        """
     641        L{_ChunkedTransferDecoder.dataReceived} disregards chunk-extension
     642        fields, even when they are long.
     643        """
     644        L = []
     645        p = http._ChunkedTransferDecoder(L.append, None)
     646        p.dataReceived('3; x-foo-long-long-long=bar-pretty-long\r\nabc\r\n')
     647        self.assertEqual(L, ['abc'])
     648
     649
     650    def test_extensionsShortDelivery(self):
     651        """
     652        L{_ChunkedTransferDecoder.dataReceived} disregards chunk-extension
     653        fields, even when delivered in multiple calls.
     654        """
     655        L = []
     656        p = http._ChunkedTransferDecoder(L.append, None)
     657        for s in '3; x-foo-long-long-long=bar-pretty-long\r\nabc\r\n':
     658            p.dataReceived(s)
     659        self.assertEqual(L, ['a', 'b', 'c'])
     660
     661
     662    def test_extensionsShortDeliveryVariant(self):
     663        """
     664        L{_ChunkedTransferDecoder.dataReceived} disregards chunk-extension
     665        fields, even when delivered in multiple calls, with the first call including
     666        the semicolon and space.
     667        """
     668        L = []
     669        p = http._ChunkedTransferDecoder(L.append, None)
     670        p.dataReceived('3; ')
     671        for s in 'x-foo-long-long-long=bar-pretty-long\r\nabc\r\n':
     672            p.dataReceived(s)
     673        self.assertEqual(L, ['a', 'b', 'c'])
     674
     675
     676    def test_extensionsShortDeliveryTwoBytes(self):
     677        """
     678        L{_ChunkedTransferDecoder.dataReceived} disregards chunk-extension
     679        fields, even when the chunk requires two bytes of chunk-length bytes to
     680        represent the length.
     681        """
     682        L = []
     683        p = http._ChunkedTransferDecoder(L.append, None)
     684        # 0x14 == 20
     685        for s in '14; x-foo-long-long-long=bar-pretty-long\r\n'+('a'*20)+'\r\n':
     686            p.dataReceived(s)
     687        self.assertEqual(L, ['a']*20)
     688
     689
    500690    def test_finish(self):
    501691        """
    502692        L{_ChunkedTransferDecoder.dataReceived} interprets a zero-length