Ticket #3795: secure-chunked-3795-4.diff

File secure-chunked-3795-4.diff, 13.3 KB (added by ivank, 7 years ago)

the only change over patch 3 is to fix one major bug with padded negative chunk lengths

  • twisted/web/http.py

    === modified file 'twisted/web/http.py'
     
    99
    1010Future Plans:
    1111 - HTTP client support will at some point be refactored to support HTTP/1.1.
    12  - Accept chunked data from clients in server.
    1312 - Other missing HTTP features from the RFC.
    1413
    1514Maintainer: Itamar Shtull-Trauring
     
    3635from twisted.python import log
    3736try: # try importing the fast, C version
    3837    from twisted.protocols._c_urlarg import unquote
     38    unquote # shut up pyflakes
    3939except ImportError:
    4040    from urllib import unquote
    4141
     
    12791279    Protocol for decoding I{chunked} Transfer-Encoding, as defined by RFC 2616,
    12801280    section 3.6.1.  This protocol can interpret the contents of a request or
    12811281    response body which uses the I{chunked} Transfer-Encoding.  It cannot
    1282     interpret any of the rest of the HTTP protocol.
     1282    interpret any of the rest of the HTTP protocol.  It ignores trailers.
    12831283
    12841284    It may make sense for _ChunkedTransferDecoder to be an actual IProtocol
    12851285    implementation.  Currently, the only user of this class will only ever
     
    12981298    @ivar finishCallback: A one-argument callable which will be invoked when
    12991299        the terminal chunk is received.  It will be invoked with all bytes
    13001300        which were delivered to this protocol which came after the terminal
    1301         chunk.
     1301        chunk.  These bytes are *not* the trailer; they might be the beginning
     1302        of the next request or response.
    13021303
    13031304    @ivar length: Counter keeping track of how many more bytes in a chunk there
    13041305        are to receive.
     
    13151316        will be accepted.
    13161317    """
    13171318    state = 'chunk-length'
    1318     finish = False
    13191319
    13201320    def __init__(self, dataCallback, finishCallback):
    13211321        self.dataCallback = dataCallback
    13221322        self.finishCallback = finishCallback
    13231323        self._buffer = ''
     1324       
     1325        # While an HTTP/1.1 chunk has no size limit in the specification, a
     1326        # reasonable limit must be established to prevent untrusted input from
     1327        # causing excessive string concatenation in the parser. A limit of 17 bytes
     1328        # (max FFFFFFFFFFFFFFFFF) can support chunks up to 2**68-1 bytes.
     1329        self._maximumChunkSizeStringLength = 17
    13241330
    13251331
    13261332    def dataReceived(self, data):
     
    13351341                if '\r\n' in data:
    13361342                    line, rest = data.split('\r\n', 1)
    13371343                    parts = line.split(';')
    1338                     self.length = int(parts[0], 16)
     1344                    chunkSizeString = parts[0].strip()
     1345                    # HEX in RFC 2616 section 2.2 does not include the minus
     1346                    # sign; negative numbers and negative zero are not allowed.
     1347                    if chunkSizeString[0] == '-':
     1348                        raise RuntimeError(
     1349                            "_ChunkedTransferDecoder.dataReceived received "
     1350                            "negative chunk length in parts %s" % (repr(parts),))
     1351                    if len(chunkSizeString) > self._maximumChunkSizeStringLength:
     1352                        raise RuntimeError(
     1353                            "_ChunkedTransferDecoder.dataReceived received "
     1354                            "too-long chunk length in parts %s" % (repr(parts),))
     1355                    try:
     1356                        self.length = int(chunkSizeString, 16)
     1357                    except ValueError:
     1358                        raise RuntimeError(
     1359                            "_ChunkedTransferDecoder.dataReceived received "
     1360                            "unparsable chunk length in parts %s" % (repr(parts),))
    13391361                    if self.length == 0:
    13401362                        self.state = 'trailer'
    1341                         self.finish = True
    13421363                    else:
    13431364                        self.state = 'body'
    13441365                    data = rest
    13451366                else:
     1367                    # Throw away HTTP/1.1 chunk-extensions every time,
     1368                    # but keep the semicolon so that additional chunk-extension
     1369                    # data doesn't get interpreted as part of the chunk-length.
     1370                    if ';' in data:
     1371                        reattachCR = (data[-1] == '\r')
     1372                        data = data[:data.find(';')+1].strip()
     1373                        if reattachCR:
     1374                            data += '\r'
     1375                        extraByte = 1
     1376                    else:
     1377                        extraByte = 0
     1378
     1379                    if len(data) > (self._maximumChunkSizeStringLength + extraByte):
     1380                        raise RuntimeError(
     1381                            "_ChunkedTransferDecoder.dataReceived buffered "
     1382                            "too-long chunk length %s" % (repr(data),))
    13461383                    self._buffer = data
    13471384                    data = ''
    1348             elif self.state == 'trailer':
     1385            elif self.state == 'crlf':
    13491386                if data.startswith('\r\n'):
    13501387                    data = data[2:]
    1351                     if self.finish:
    1352                         self.state = 'finished'
    1353                         self.finishCallback(data)
    1354                         data = ''
    1355                     else:
    1356                         self.state = 'chunk-length'
    1357                 else:
     1388                    self.state = 'chunk-length'
     1389                elif data == '\r':
    13581390                    self._buffer = data
    13591391                    data = ''
     1392                else:
     1393                    raise RuntimeError(
     1394                        "_ChunkedTransferDecoder.dataReceived was looking for "
     1395                        "CRLF, not %s" % (repr(data),))
     1396            elif self.state == 'trailer':
     1397                # The goal is to throw away as much of the trailer as
     1398                # possible every time, while hoping to get the end-of-trailer.
     1399                CRLF_at = data.find('\r\n')
     1400                if CRLF_at != -1:
     1401                    # The *first* '\r\n' ends the trailer.
     1402                    data = data[CRLF_at+2:]
     1403                    self.state = 'finished'
     1404                    self.finishCallback(data)
     1405                elif data[-1] == '\r':
     1406                    self._buffer = data[-1]
     1407                data = ''   
    13601408            elif self.state == 'body':
    13611409                if len(data) >= self.length:
    13621410                    chunk, data = data[:self.length], data[self.length:]
    13631411                    self.dataCallback(chunk)
    1364                     self.state = 'trailer'
     1412                    self.state = 'crlf'
    13651413                elif len(data) < self.length:
    13661414                    self.length -= len(data)
    13671415                    self.dataCallback(data)
  • twisted/web/test/test_http.py

    === modified file 'twisted/web/test/test_http.py'
     
    527527        """
    528528        p = http._ChunkedTransferDecoder(None, lambda bytes: None)
    529529        p.dataReceived('0\r\n\r\n')
    530         self.assertRaises(RuntimeError, p.dataReceived, 'hello')
    531 
     530        exc = self.assertRaises(RuntimeError, p.dataReceived, 'hello')
     531        self.assertEqual(
     532            str(exc),
     533            "_ChunkedTransferDecoder.dataReceived called after last "
     534            "chunk was processed")
     535           
    532536
    533537    def test_earlyConnectionLose(self):
    534538        """
     
    574578        self.assertEqual(successes, [True])
    575579
    576580
     581    def test_trailerUsesNoMemory(self):
     582        """
     583        L{_ChunkedTransferDecoder.dataReceived} does not waste memory
     584        buffering pieces of the trailer, which is always ignored anyway.
     585
     586        This test is very implementation-specific because the parser exhibits
     587        no public behavior while ignoring the trailer.
     588        """
     589        L = []
     590        finished = []
     591        p = http._ChunkedTransferDecoder(L.append, finished.append)
     592        p.dataReceived('3\r\nabc\r\n0\r\nTRAILER')
     593        self.assertEqual(len(p._buffer), 0)
     594        p.dataReceived('MORE TRAILER')
     595        self.assertEqual(len(p._buffer), 0)
     596        p.dataReceived('Here comes a CR: \r')
     597        self.assertEqual(len(p._buffer), 1)
     598        p.dataReceived('But no newline!')
     599        self.assertEqual(len(p._buffer), 0)
     600        p.dataReceived('Really finish the trailer now: \r\n')
     601        self.assertEqual(len(p._buffer), 0)
     602        self.assertEqual(L, ['abc'])
     603        self.assertEqual(finished, [''])
     604
     605
     606    def test_chunkExtensionsUseNoMemory(self):
     607        """
     608        L{_ChunkedTransferDecoder.dataReceived} does not waste memory
     609        buffering pieces of chunk extensions, which are always ignored anyway.
     610
     611        This test is very implementation-specific because the parser exhibits
     612        no public behavior while ignoring the chunk extensions.
     613        """
     614        L = []
     615        finished = []
     616        p = http._ChunkedTransferDecoder(L.append, finished.append)
     617        p.dataReceived('3\r\nabc\r\n4; hello=yes')
     618        originalLength = len(p._buffer)
     619        # feed it some more ignored chunk-extension
     620        p.dataReceived('-still-ignored')
     621        self.assertEqual(len(p._buffer), originalLength)
     622
     623
     624    def test_limitedChunkLengthBuffering(self):
     625        """
     626        L{_ChunkedTransferDecoder.dataReceived} does not allow input
     627        to endlessly fill its buffer with a chunk length string.
     628        """
     629        L = []
     630        p = http._ChunkedTransferDecoder(L.append, None)
     631        max = p._maximumChunkSizeStringLength
     632
     633        p.dataReceived('2\r\nab\r\n')
     634        exc = self.assertRaises(RuntimeError, lambda: p.dataReceived('3' * (max+1)))
     635        self.assertEqual(
     636            str(exc),
     637            "_ChunkedTransferDecoder.dataReceived buffered too-long "
     638            "chunk length '333333333333333333'")
     639
     640
     641    def test_limitedChunkLengthBufferingShort(self):
     642        """
     643        L{_ChunkedTransferDecoder.dataReceived} does not allow input
     644        to endlessly fill its buffer with a chunk length string, even when
     645        the data is delivered with multiple calls.
     646        """
     647        L = []
     648        p = http._ChunkedTransferDecoder(L.append, None)
     649        max = p._maximumChunkSizeStringLength
     650
     651        p.dataReceived('2\r\nab\r\n')
     652        for s in '3' * max:
     653            p.dataReceived(s)
     654        exc = self.assertRaises(RuntimeError, lambda: p.dataReceived('3' * 1))
     655        self.assertEqual(
     656            str(exc),
     657            "_ChunkedTransferDecoder.dataReceived buffered too-long "
     658            "chunk length '333333333333333333'")
     659
     660
     661    def test_chunkLengthNotTooLong(self):
     662        """
     663
     664        """
     665        L = []
     666        p = http._ChunkedTransferDecoder(L.append, None)
     667        max = p._maximumChunkSizeStringLength
     668
     669        p.dataReceived('2\r\nab\r\n')
     670
     671        chunkLenString = ('3' * (max+1))
     672        exc = self.assertRaises(RuntimeError,
     673            lambda: p.dataReceived(chunkLenString + '\r\n'))
     674           
     675        self.assertEqual(
     676            str(exc),
     677            "_ChunkedTransferDecoder.dataReceived received "
     678            "too-long chunk length in parts %s" % (repr([chunkLenString]),))
     679
     680
     681    def test_chunkLengthSemicolonMath(self):
     682        """
     683        L{_ChunkedTransferDecoder.dataReceived} doesn't include
     684        the length of the semicolon or chunk-extension data when
     685        determining the length of the chunk-length bytes.
     686        """
     687        L = []
     688        p = http._ChunkedTransferDecoder(L.append, None)
     689        max = p._maximumChunkSizeStringLength
     690
     691        p.dataReceived((('3' * (max)) + '; long-extension-completely-ignored=yes'))
     692
     693
     694    def test_chunkLengthNotUnparsable(self):
     695        """
     696
     697        """
     698        L = []
     699        p = http._ChunkedTransferDecoder(L.append, None)
     700
     701        p.dataReceived('2\r\nab\r\n')
     702
     703        chunkLenString = ('G')
     704        exc = self.assertRaises(RuntimeError,
     705            lambda: p.dataReceived(chunkLenString + '\r\n'))
     706
     707        self.assertEqual(
     708            str(exc),
     709            "_ChunkedTransferDecoder.dataReceived received "
     710            "unparsable chunk length in parts %s" % (repr([chunkLenString]),))
     711
     712
     713    def test_chunkLengthNotNegative(self):
     714        """
     715
     716        """
     717        L = []
     718        p = http._ChunkedTransferDecoder(L.append, None)
     719
     720        p.dataReceived('2\r\nab\r\n')
     721        exc = self.assertRaises(RuntimeError, lambda: p.dataReceived('-1\r\n'))
     722        self.assertEqual(
     723            str(exc),
     724            "_ChunkedTransferDecoder.dataReceived received "
     725            "negative chunk length in parts %s" % (repr(['-1']),))
     726
     727
     728    def test_chunkLengthNotNegativeWithPadding(self):
     729        """
     730
     731        """
     732        L = []
     733        p = http._ChunkedTransferDecoder(L.append, None)
     734
     735        p.dataReceived('2\r\nab\r\n')
     736        exc = self.assertRaises(RuntimeError, lambda: p.dataReceived(' -1\r\n'))
     737        self.assertEqual(
     738            str(exc),
     739            "_ChunkedTransferDecoder.dataReceived received "
     740            "negative chunk length in parts %s" % (repr([' -1']),))
     741
     742
     743    def test_chunkLengthNotNegativeZero(self):
     744        """
     745
     746        """
     747        L = []
     748        p = http._ChunkedTransferDecoder(L.append, None)
     749
     750        p.dataReceived('2\r\nab\r\n')
     751        exc = self.assertRaises(RuntimeError, lambda: p.dataReceived('-0\r\n'))
     752        self.assertEqual(
     753            str(exc),
     754            "_ChunkedTransferDecoder.dataReceived received "
     755            "negative chunk length in parts %s" % (repr(['-0']),))
     756
     757
    577758
    578759class ChunkingTestCase(unittest.TestCase):
    579760