Ticket #886: t.web._newclient_support_bare_LF.patch

File t.web._newclient_support_bare_LF.patch, 5.8 KB (added by ivank, 7 years ago)

support for bare LF newlines sent by broken servers

  • twisted/web/_newclient.py

    === added file 'DO_NOT_MERGE_,FOR_TWISTED_ONLY'
    === modified file 'twisted/web/_newclient.py' (properties changed: -x to +x)
     
    209209    #
    210210    # -exarkun
    211211
     212    delimiter = '\n'
     213
    212214    CONNECTION_CONTROL_HEADERS = set([
    213215            'content-length', 'connection', 'keep-alive', 'te', 'trailers',
    214216            'transfer-encoding', 'upgrade', 'proxy-connection'])
     
    237239        """
    238240        Handle one line from a response.
    239241        """
     242
     243        # some servers like http://news.ycombinator.com/ return status lines
     244        # and http headers delimited by \n instead of \r\n
     245        if line[-1:] == '\r':
     246            line = line[:-1]
     247
    240248        if self.state == STATUS:
    241249            self.statusReceived(line)
    242250            self.state = HEADER
  • twisted/web/test/test_newclient.py

    === modified file 'twisted/web/test/test_newclient.py' (properties changed: -x to +x)
     
    150150
    151151
    152152
    153 class HTTPParserTests(TestCase):
    154     """
    155     Tests for L{HTTPParser} which is responsible for the bulk of the task of
    156     parsing HTTP bytes.
    157     """
     153class _HTTPParserTests(object):
     154    """
     155    Base test class for L{HTTPParser} which is responsible for the bulk of
     156    the task of parsing HTTP bytes.
     157    """
     158
     159    sep = None
     160
    158161    def test_statusCallback(self):
    159162        """
    160163        L{HTTPParser} calls its C{statusReceived} method when it receives a
     
    165168        protocol.statusReceived = status.append
    166169        protocol.makeConnection(StringTransport())
    167170        self.assertEqual(protocol.state, STATUS)
    168         protocol.dataReceived('HTTP/1.1 200 OK\r\n')
     171        protocol.dataReceived('HTTP/1.1 200 OK' + self.sep)
    169172        self.assertEqual(status, ['HTTP/1.1 200 OK'])
    170173        self.assertEqual(protocol.state, HEADER)
    171174
     
    175178        protocol = HTTPParser()
    176179        protocol.headerReceived = header.__setitem__
    177180        protocol.makeConnection(StringTransport())
    178         protocol.dataReceived('HTTP/1.1 200 OK\r\n')
     181        protocol.dataReceived('HTTP/1.1 200 OK' + self.sep)
    179182        return header, protocol
    180183
    181184
     
    185188        header.
    186189        """
    187190        header, protocol = self._headerTestSetup()
    188         protocol.dataReceived('X-Foo:bar\r\n')
     191        protocol.dataReceived('X-Foo:bar' + self.sep)
    189192        # Cannot tell it's not a continue header until the next line arrives
    190193        # and is not a continuation
    191         protocol.dataReceived('\r\n')
     194        protocol.dataReceived(self.sep)
    192195        self.assertEqual(header, {'X-Foo': 'bar'})
    193196        self.assertEqual(protocol.state, BODY)
    194197
     
    199202        C{headerReceived} with the entire value once it is received.
    200203        """
    201204        header, protocol = self._headerTestSetup()
    202         protocol.dataReceived('X-Foo: bar\r\n')
    203         protocol.dataReceived(' baz\r\n')
    204         protocol.dataReceived('\tquux\r\n')
    205         protocol.dataReceived('\r\n')
     205        protocol.dataReceived('X-Foo: bar' + self.sep)
     206        protocol.dataReceived(' baz' + self.sep)
     207        protocol.dataReceived('\tquux' + self.sep)
     208        protocol.dataReceived(self.sep)
    206209        self.assertEqual(header, {'X-Foo': 'bar baz\tquux'})
    207210        self.assertEqual(protocol.state, BODY)
    208211
     
    213216        value passed to the C{headerReceived} callback.
    214217        """
    215218        header, protocol = self._headerTestSetup()
    216         value = ' \t \r\n bar \t\r\n \t\r\n'
     219        value = ' \t %(sep)s bar \t%(sep)s \t%(sep)s' % dict(sep=self.sep)
    217220        protocol.dataReceived('X-Bar:' + value)
    218221        protocol.dataReceived('X-Foo:' + value)
    219         protocol.dataReceived('\r\n')
     222        protocol.dataReceived(self.sep)
    220223        self.assertEqual(header, {'X-Foo': 'bar',
    221224                                  'X-Bar': 'bar'})
    222225
     
    232235            called.append(protocol.state)
    233236            protocol.state = STATUS
    234237        protocol.allHeadersReceived = allHeadersReceived
    235         protocol.dataReceived('\r\n')
     238        protocol.dataReceived(self.sep)
    236239        self.assertEqual(called, [HEADER])
    237240        self.assertEqual(protocol.state, STATUS)
    238241
     
    243246        C{headerReceived}.
    244247        """
    245248        header, protocol = self._headerTestSetup()
    246         protocol.dataReceived('\r\n')
     249        protocol.dataReceived(self.sep)
    247250        self.assertEqual(header, {})
    248251        self.assertEqual(protocol.state, BODY)
    249252
     
    255258        """
    256259        protocol = HTTPParser()
    257260        protocol.makeConnection(StringTransport())
    258         protocol.dataReceived('HTTP/1.1 200 OK\r\n')
    259         protocol.dataReceived('X-Foo: bar\r\n')
    260         protocol.dataReceived('X-Foo: baz\r\n')
    261         protocol.dataReceived('\r\n')
    262         self.assertEqual(
    263             list(protocol.headers.getAllRawHeaders()),
    264             [('X-Foo', ['bar', 'baz'])])
     261        protocol.dataReceived('HTTP/1.1 200 OK' + self.sep)
     262        protocol.dataReceived('X-Foo: bar' + self.sep)
     263        protocol.dataReceived('X-Foo: baz' + self.sep)
     264        protocol.dataReceived(self.sep)
     265        expected = [('X-Foo', ['bar', 'baz'])]
     266        self.assertEqual(expected, list(protocol.headers.getAllRawHeaders()))
    265267
    266268
    267269    def test_connectionControlHeaders(self):
     
    298300
    299301
    300302
     303class HTTPParserTestsRFCComplaintDelimeter(_HTTPParserTests, TestCase):
     304    """
     305    L{_HTTPParserTests} using standard CR LF newlines.
     306    """
     307    sep = '\r\n'
     308
     309
     310
     311class HTTPParserTestsNonRFCComplaintDelimeter(_HTTPParserTests, TestCase):
     312    """
     313    L{_HTTPParserTests} using bare LF newlines.
     314    """
     315    sep = '\n'
     316   
     317
     318
    301319class HTTPClientParserTests(TestCase):
    302320    """
    303321    Tests for L{HTTPClientParser} which is responsible for parsing HTTP