Ticket #3795: secure-chunked-3795-5.diff

File secure-chunked-3795-5.diff, 13.9 KB (added by ivank, 7 years ago)

only change over patch 4 is to exercise the reattachCR logic

  • twisted/web/http.py

    === modified file 'twisted/web/http.py'
     
    99
    1010Future Plans:
    1111 - HTTP client support will at some point be refactored to support HTTP/1.1.
    12  - Accept chunked data from clients in server.
    1312 - Other missing HTTP features from the RFC.
    1413
    1514Maintainer: Itamar Shtull-Trauring
     
    3635from twisted.python import log
    3736try: # try importing the fast, C version
    3837    from twisted.protocols._c_urlarg import unquote
     38    unquote # shut up pyflakes
    3939except ImportError:
    4040    from urllib import unquote
    4141
     
    12791279    Protocol for decoding I{chunked} Transfer-Encoding, as defined by RFC 2616,
    12801280    section 3.6.1.  This protocol can interpret the contents of a request or
    12811281    response body which uses the I{chunked} Transfer-Encoding.  It cannot
    1282     interpret any of the rest of the HTTP protocol.
     1282    interpret any of the rest of the HTTP protocol.  It ignores trailers.
    12831283
    12841284    It may make sense for _ChunkedTransferDecoder to be an actual IProtocol
    12851285    implementation.  Currently, the only user of this class will only ever
     
    12981298    @ivar finishCallback: A one-argument callable which will be invoked when
    12991299        the terminal chunk is received.  It will be invoked with all bytes
    13001300        which were delivered to this protocol which came after the terminal
    1301         chunk.
     1301        chunk.  These bytes are *not* the trailer; they might be the beginning
     1302        of the next request or response.
    13021303
    13031304    @ivar length: Counter keeping track of how many more bytes in a chunk there
    13041305        are to receive.
     
    13151316        will be accepted.
    13161317    """
    13171318    state = 'chunk-length'
    1318     finish = False
    13191319
    13201320    def __init__(self, dataCallback, finishCallback):
    13211321        self.dataCallback = dataCallback
    13221322        self.finishCallback = finishCallback
    13231323        self._buffer = ''
     1324       
     1325        # While an HTTP/1.1 chunk has no size limit in the specification, a
     1326        # reasonable limit must be established to prevent untrusted input from
     1327        # causing excessive string concatenation in the parser. A limit of 17 bytes
     1328        # (max FFFFFFFFFFFFFFFFF) can support chunks up to 2**68-1 bytes.
     1329        self._maximumChunkSizeStringLength = 17
    13241330
    13251331
    13261332    def dataReceived(self, data):
     
    13351341                if '\r\n' in data:
    13361342                    line, rest = data.split('\r\n', 1)
    13371343                    parts = line.split(';')
    1338                     self.length = int(parts[0], 16)
     1344                    chunkSizeString = parts[0].strip()
     1345                    # HEX in RFC 2616 section 2.2 does not include the minus
     1346                    # sign; negative numbers and negative zero are not allowed.
     1347                    if chunkSizeString[0] == '-':
     1348                        raise RuntimeError(
     1349                            "_ChunkedTransferDecoder.dataReceived received "
     1350                            "negative chunk length in parts %s" % (repr(parts),))
     1351                    if len(chunkSizeString) > self._maximumChunkSizeStringLength:
     1352                        raise RuntimeError(
     1353                            "_ChunkedTransferDecoder.dataReceived received "
     1354                            "too-long chunk length in parts %s" % (repr(parts),))
     1355                    try:
     1356                        self.length = int(chunkSizeString, 16)
     1357                    except ValueError:
     1358                        raise RuntimeError(
     1359                            "_ChunkedTransferDecoder.dataReceived received "
     1360                            "unparsable chunk length in parts %s" % (repr(parts),))
    13391361                    if self.length == 0:
    13401362                        self.state = 'trailer'
    1341                         self.finish = True
    13421363                    else:
    13431364                        self.state = 'body'
    13441365                    data = rest
    13451366                else:
     1367                    # Throw away HTTP/1.1 chunk-extensions every time,
     1368                    # but keep the semicolon so that additional chunk-extension
     1369                    # data doesn't get interpreted as part of the chunk-length.
     1370                    if ';' in data:
     1371                        reattachCR = (data[-1] == '\r')
     1372                        data = data[:data.find(';')+1].strip()
     1373                        if reattachCR:
     1374                            data += '\r'
     1375                        extraByte = 1
     1376                    else:
     1377                        extraByte = 0
     1378
     1379                    if len(data) > (self._maximumChunkSizeStringLength + extraByte):
     1380                        raise RuntimeError(
     1381                            "_ChunkedTransferDecoder.dataReceived buffered "
     1382                            "too-long chunk length %s" % (repr(data),))
    13461383                    self._buffer = data
    13471384                    data = ''
    1348             elif self.state == 'trailer':
     1385            elif self.state == 'crlf':
    13491386                if data.startswith('\r\n'):
    13501387                    data = data[2:]
    1351                     if self.finish:
    1352                         self.state = 'finished'
    1353                         self.finishCallback(data)
    1354                         data = ''
    1355                     else:
    1356                         self.state = 'chunk-length'
    1357                 else:
     1388                    self.state = 'chunk-length'
     1389                elif data == '\r':
    13581390                    self._buffer = data
    13591391                    data = ''
     1392                else:
     1393                    raise RuntimeError(
     1394                        "_ChunkedTransferDecoder.dataReceived was looking for "
     1395                        "CRLF, not %s" % (repr(data),))
     1396            elif self.state == 'trailer':
     1397                # The goal is to throw away as much of the trailer as
     1398                # possible every time, while hoping to get the end-of-trailer.
     1399                CRLF_at = data.find('\r\n')
     1400                if CRLF_at != -1:
     1401                    # The *first* '\r\n' ends the trailer.
     1402                    data = data[CRLF_at+2:]
     1403                    self.state = 'finished'
     1404                    self.finishCallback(data)
     1405                elif data[-1] == '\r':
     1406                    self._buffer = data[-1]
     1407                data = ''   
    13601408            elif self.state == 'body':
    13611409                if len(data) >= self.length:
    13621410                    chunk, data = data[:self.length], data[self.length:]
    13631411                    self.dataCallback(chunk)
    1364                     self.state = 'trailer'
     1412                    self.state = 'crlf'
    13651413                elif len(data) < self.length:
    13661414                    self.length -= len(data)
    13671415                    self.dataCallback(data)
  • twisted/web/test/test_http.py

    === modified file 'twisted/web/test/test_http.py'
     
    497497        self.assertEqual(L, ['abc'])
    498498
    499499
     500    def test_extensionsShort(self):
     501        """
     502        L{_ChunkedTransferDecoder.dataReceived} disregards chunk-extension
     503        fields, even when sent one byte at a time.
     504
     505        This should exercise the reattachCR logic in the parser.
     506        """
     507        L = []
     508        p = http._ChunkedTransferDecoder(L.append, None)
     509        for s in '3; x-foo=bar\r\nabc\r\n':
     510            p.dataReceived(s)
     511        self.assertEqual(L, ['a', 'b', 'c'])
     512
     513
    500514    def test_finish(self):
    501515        """
    502516        L{_ChunkedTransferDecoder.dataReceived} interprets a zero-length
     
    527541        """
    528542        p = http._ChunkedTransferDecoder(None, lambda bytes: None)
    529543        p.dataReceived('0\r\n\r\n')
    530         self.assertRaises(RuntimeError, p.dataReceived, 'hello')
    531 
     544        exc = self.assertRaises(RuntimeError, p.dataReceived, 'hello')
     545        self.assertEqual(
     546            str(exc),
     547            "_ChunkedTransferDecoder.dataReceived called after last "
     548            "chunk was processed")
     549           
    532550
    533551    def test_earlyConnectionLose(self):
    534552        """
     
    574592        self.assertEqual(successes, [True])
    575593
    576594
     595    def test_trailerUsesNoMemory(self):
     596        """
     597        L{_ChunkedTransferDecoder.dataReceived} does not waste memory
     598        buffering pieces of the trailer, which is always ignored anyway.
     599
     600        This test is very implementation-specific because the parser exhibits
     601        no public behavior while ignoring the trailer.
     602        """
     603        L = []
     604        finished = []
     605        p = http._ChunkedTransferDecoder(L.append, finished.append)
     606        p.dataReceived('3\r\nabc\r\n0\r\nTRAILER')
     607        self.assertEqual(len(p._buffer), 0)
     608        p.dataReceived('MORE TRAILER')
     609        self.assertEqual(len(p._buffer), 0)
     610        p.dataReceived('Here comes a CR: \r')
     611        self.assertEqual(len(p._buffer), 1)
     612        p.dataReceived('But no newline!')
     613        self.assertEqual(len(p._buffer), 0)
     614        p.dataReceived('Really finish the trailer now: \r\n')
     615        self.assertEqual(len(p._buffer), 0)
     616        self.assertEqual(L, ['abc'])
     617        self.assertEqual(finished, [''])
     618
     619
     620    def test_chunkExtensionsUseNoMemory(self):
     621        """
     622        L{_ChunkedTransferDecoder.dataReceived} does not waste memory
     623        buffering pieces of chunk extensions, which are always ignored anyway.
     624
     625        This test is very implementation-specific because the parser exhibits
     626        no public behavior while ignoring the chunk extensions.
     627        """
     628        L = []
     629        finished = []
     630        p = http._ChunkedTransferDecoder(L.append, finished.append)
     631        p.dataReceived('3\r\nabc\r\n4; hello=yes')
     632        originalLength = len(p._buffer)
     633        # feed it some more ignored chunk-extension
     634        p.dataReceived('-still-ignored')
     635        self.assertEqual(len(p._buffer), originalLength)
     636
     637
     638    def test_limitedChunkLengthBuffering(self):
     639        """
     640        L{_ChunkedTransferDecoder.dataReceived} does not allow input
     641        to endlessly fill its buffer with a chunk length string.
     642        """
     643        L = []
     644        p = http._ChunkedTransferDecoder(L.append, None)
     645        max = p._maximumChunkSizeStringLength
     646
     647        p.dataReceived('2\r\nab\r\n')
     648        exc = self.assertRaises(RuntimeError, lambda: p.dataReceived('3' * (max+1)))
     649        self.assertEqual(
     650            str(exc),
     651            "_ChunkedTransferDecoder.dataReceived buffered too-long "
     652            "chunk length '333333333333333333'")
     653
     654
     655    def test_limitedChunkLengthBufferingShort(self):
     656        """
     657        L{_ChunkedTransferDecoder.dataReceived} does not allow input
     658        to endlessly fill its buffer with a chunk length string, even when
     659        the data is delivered with multiple calls.
     660        """
     661        L = []
     662        p = http._ChunkedTransferDecoder(L.append, None)
     663        max = p._maximumChunkSizeStringLength
     664
     665        p.dataReceived('2\r\nab\r\n')
     666        for s in '3' * max:
     667            p.dataReceived(s)
     668        exc = self.assertRaises(RuntimeError, lambda: p.dataReceived('3' * 1))
     669        self.assertEqual(
     670            str(exc),
     671            "_ChunkedTransferDecoder.dataReceived buffered too-long "
     672            "chunk length '333333333333333333'")
     673
     674
     675    def test_chunkLengthNotTooLong(self):
     676        """
     677
     678        """
     679        L = []
     680        p = http._ChunkedTransferDecoder(L.append, None)
     681        max = p._maximumChunkSizeStringLength
     682
     683        p.dataReceived('2\r\nab\r\n')
     684
     685        chunkLenString = ('3' * (max+1))
     686        exc = self.assertRaises(RuntimeError,
     687            lambda: p.dataReceived(chunkLenString + '\r\n'))
     688           
     689        self.assertEqual(
     690            str(exc),
     691            "_ChunkedTransferDecoder.dataReceived received "
     692            "too-long chunk length in parts %s" % (repr([chunkLenString]),))
     693
     694
     695    def test_chunkLengthSemicolonMath(self):
     696        """
     697        L{_ChunkedTransferDecoder.dataReceived} doesn't include
     698        the length of the semicolon or chunk-extension data when
     699        determining the length of the chunk-length bytes.
     700        """
     701        L = []
     702        p = http._ChunkedTransferDecoder(L.append, None)
     703        max = p._maximumChunkSizeStringLength
     704
     705        p.dataReceived((('3' * (max)) + '; long-extension-completely-ignored=yes'))
     706
     707
     708    def test_chunkLengthNotUnparsable(self):
     709        """
     710
     711        """
     712        L = []
     713        p = http._ChunkedTransferDecoder(L.append, None)
     714
     715        p.dataReceived('2\r\nab\r\n')
     716
     717        chunkLenString = ('G')
     718        exc = self.assertRaises(RuntimeError,
     719            lambda: p.dataReceived(chunkLenString + '\r\n'))
     720
     721        self.assertEqual(
     722            str(exc),
     723            "_ChunkedTransferDecoder.dataReceived received "
     724            "unparsable chunk length in parts %s" % (repr([chunkLenString]),))
     725
     726
     727    def test_chunkLengthNotNegative(self):
     728        """
     729
     730        """
     731        L = []
     732        p = http._ChunkedTransferDecoder(L.append, None)
     733
     734        p.dataReceived('2\r\nab\r\n')
     735        exc = self.assertRaises(RuntimeError, lambda: p.dataReceived('-1\r\n'))
     736        self.assertEqual(
     737            str(exc),
     738            "_ChunkedTransferDecoder.dataReceived received "
     739            "negative chunk length in parts %s" % (repr(['-1']),))
     740
     741
     742    def test_chunkLengthNotNegativeWithPadding(self):
     743        """
     744
     745        """
     746        L = []
     747        p = http._ChunkedTransferDecoder(L.append, None)
     748
     749        p.dataReceived('2\r\nab\r\n')
     750        exc = self.assertRaises(RuntimeError, lambda: p.dataReceived(' -1\r\n'))
     751        self.assertEqual(
     752            str(exc),
     753            "_ChunkedTransferDecoder.dataReceived received "
     754            "negative chunk length in parts %s" % (repr([' -1']),))
     755
     756
     757    def test_chunkLengthNotNegativeZero(self):
     758        """
     759
     760        """
     761        L = []
     762        p = http._ChunkedTransferDecoder(L.append, None)
     763
     764        p.dataReceived('2\r\nab\r\n')
     765        exc = self.assertRaises(RuntimeError, lambda: p.dataReceived('-0\r\n'))
     766        self.assertEqual(
     767            str(exc),
     768            "_ChunkedTransferDecoder.dataReceived received "
     769            "negative chunk length in parts %s" % (repr(['-0']),))
     770
     771
    577772
    578773class ChunkingTestCase(unittest.TestCase):
    579774