Ticket #886: t.web._newclient_support_bare_LF.patch

twisted/web/_newclient.py

@@ -209 +209 @@
     #
     # -exarkun
 
+    delimiter = '\n'
+
     CONNECTION_CONTROL_HEADERS = set([
             'content-length', 'connection', 'keep-alive', 'te', 'trailers',
             'transfer-encoding', 'upgrade', 'proxy-connection'])
@@ -237 +239 @@
         """
         Handle one line from a response.
         """
+
+        # some servers like http://news.ycombinator.com/ return status lines
+        # and http headers delimited by \n instead of \r\n
+        if line[-1:] == '\r':
+            line = line[:-1]
+
         if self.state == STATUS:
             self.statusReceived(line)
             self.state = HEADER

twisted/web/test/test_newclient.py

@@ -150 +150 @@
 
 
 
-class HTTPParserTests(TestCase):
-    """
-    Tests for L{HTTPParser} which is responsible for the bulk of the task of
-    parsing HTTP bytes.
-    """
+class _HTTPParserTests(object):
+    """
+    Base test class for L{HTTPParser} which is responsible for the bulk of
+    the task of parsing HTTP bytes.
+    """
+
+    sep = None
+
     def test_statusCallback(self):
         """
         L{HTTPParser} calls its C{statusReceived} method when it receives a
@@ -165 +168 @@
         protocol.statusReceived = status.append
         protocol.makeConnection(StringTransport())
         self.assertEqual(protocol.state, STATUS)
-        protocol.dataReceived('HTTP/1.1 200 OK\r\n')
+        protocol.dataReceived('HTTP/1.1 200 OK' + self.sep)
         self.assertEqual(status, ['HTTP/1.1 200 OK'])
         self.assertEqual(protocol.state, HEADER)
 
@@ -175 +178 @@
         protocol = HTTPParser()
         protocol.headerReceived = header.__setitem__
         protocol.makeConnection(StringTransport())
-        protocol.dataReceived('HTTP/1.1 200 OK\r\n')
+        protocol.dataReceived('HTTP/1.1 200 OK' + self.sep)
         return header, protocol
 
 
@@ -185 +188 @@
         header.
         """
         header, protocol = self._headerTestSetup()
-        protocol.dataReceived('X-Foo:bar\r\n')
+        protocol.dataReceived('X-Foo:bar' + self.sep)
         # Cannot tell it's not a continue header until the next line arrives
         # and is not a continuation
-        protocol.dataReceived('\r\n')
+        protocol.dataReceived(self.sep)
         self.assertEqual(header, {'X-Foo': 'bar'})
         self.assertEqual(protocol.state, BODY)
 
@@ -199 +202 @@
         C{headerReceived} with the entire value once it is received.
         """
         header, protocol = self._headerTestSetup()
-        protocol.dataReceived('X-Foo: bar\r\n')
-        protocol.dataReceived(' baz\r\n')
-        protocol.dataReceived('\tquux\r\n')
-        protocol.dataReceived('\r\n')
+        protocol.dataReceived('X-Foo: bar' + self.sep)
+        protocol.dataReceived(' baz' + self.sep)
+        protocol.dataReceived('\tquux' + self.sep)
+        protocol.dataReceived(self.sep)
         self.assertEqual(header, {'X-Foo': 'bar baz\tquux'})
         self.assertEqual(protocol.state, BODY)
 
@@ -213 +216 @@
         value passed to the C{headerReceived} callback.
         """
         header, protocol = self._headerTestSetup()
-        value = ' \t \r\n bar \t\r\n \t\r\n'
+        value = ' \t %(sep)s bar \t%(sep)s \t%(sep)s' % dict(sep=self.sep)
         protocol.dataReceived('X-Bar:' + value)
         protocol.dataReceived('X-Foo:' + value)
-        protocol.dataReceived('\r\n')
+        protocol.dataReceived(self.sep)
         self.assertEqual(header, {'X-Foo': 'bar',
                                   'X-Bar': 'bar'})
 
@@ -232 +235 @@
             called.append(protocol.state)
             protocol.state = STATUS
         protocol.allHeadersReceived = allHeadersReceived
-        protocol.dataReceived('\r\n')
+        protocol.dataReceived(self.sep)
         self.assertEqual(called, [HEADER])
         self.assertEqual(protocol.state, STATUS)
 
@@ -243 +246 @@
         C{headerReceived}.
         """
         header, protocol = self._headerTestSetup()
-        protocol.dataReceived('\r\n')
+        protocol.dataReceived(self.sep)
         self.assertEqual(header, {})
         self.assertEqual(protocol.state, BODY)
 
@@ -255 +258 @@
         """
         protocol = HTTPParser()
         protocol.makeConnection(StringTransport())
-        protocol.dataReceived('HTTP/1.1 200 OK\r\n')
-        protocol.dataReceived('X-Foo: bar\r\n')
-        protocol.dataReceived('X-Foo: baz\r\n')
-        protocol.dataReceived('\r\n')
-        self.assertEqual(
-            list(protocol.headers.getAllRawHeaders()),
-            [('X-Foo', ['bar', 'baz'])])
+        protocol.dataReceived('HTTP/1.1 200 OK' + self.sep)
+        protocol.dataReceived('X-Foo: bar' + self.sep)
+        protocol.dataReceived('X-Foo: baz' + self.sep)
+        protocol.dataReceived(self.sep)
+        expected = [('X-Foo', ['bar', 'baz'])]
+        self.assertEqual(expected, list(protocol.headers.getAllRawHeaders()))
 
 
     def test_connectionControlHeaders(self):
@@ -298 +300 @@
 
 
 
+class HTTPParserTestsRFCComplaintDelimeter(_HTTPParserTests, TestCase):
+    """
+    L{_HTTPParserTests} using standard CR LF newlines.
+    """
+    sep = '\r\n'
+
+
+
+class HTTPParserTestsNonRFCComplaintDelimeter(_HTTPParserTests, TestCase):
+    """
+    L{_HTTPParserTests} using bare LF newlines.
+    """
+    sep = '\n'
+    
+
+
 class HTTPClientParserTests(TestCase):
     """
     Tests for L{HTTPClientParser} which is responsible for parsing HTTP