[Twisted-web] Bug in twisted.web2.client when receiving non-chunked responses of indeterminate length

[Andrew Warkentin]

Andrew Warkentin wrote:

> twisted.web2.client cuts responses short if they are not in chunked =

> encoding and do not have a Content-Length header. I have attached a =

> patch that fixes this bug. I am posting it to the list because I =

> couldn't register on the site to post it to the bug tracker.
>
The previous patch introduced anotther bug which prevented the Deferred =

returned from HTTPClient.submit from being fired if the connection was =

closed cleanly before the data was sent. I have attached an updated =

version of the patch which does not have that bug.
-------------- next part --------------
Only in TwistedWeb2-svn20071201: build
diff -r -C 3 TwistedWeb2-svn20071201.orig/twisted/web2/channel/http.py Twis=
tedWeb2-svn20071201/twisted/web2/channel/http.py
*** TwistedWeb2-svn20071201.orig/twisted/web2/channel/http.py	Sat Dec  1 04=
:10:51 2007
--- TwistedWeb2-svn20071201/twisted/web2/channel/http.py	Fri Mar 28 04:40:0=
1 2008
***************
*** 163,176 ****
              self.handleContentChunk(data[:self.length])
              extraneous =3D data[self.length:]
              channel =3D self.channel # could go away from allContentRecei=
ved.
!             if not self.chunkedIn:
!                 self.allContentReceived()
!             else:
!                 # NOTE: in chunked mode, self.length is the size of the c=
urrent chunk,
!                 # so we still have more to read.
!                 self.chunkedIn =3D 2 # Read next chunksize
              =

!             channel.setLineMode(extraneous)
  =

      def headerReceived(self, line):
          """Store this header away. Check for too much header data
--- 163,177 ----
              self.handleContentChunk(data[:self.length])
              extraneous =3D data[self.length:]
              channel =3D self.channel # could go away from allContentRecei=
ved.
!             if not (self.length is None and not self.chunkedIn and self.p=
arseCloseAsEnd):
!                 if not self.chunkedIn:
!                     self.allContentReceived()
!                 else:
!                     # NOTE: in chunked mode, self.length is the size of t=
he current chunk,
!                     # so we still have more to read.
!                     self.chunkedIn =3D 2 # Read next chunksize
              =

!                 channel.setLineMode(extraneous)
  =

      def headerReceived(self, line):
          """Store this header away. Check for too much header data
diff -r -C 3 TwistedWeb2-svn20071201.orig/twisted/web2/client/http.py Twist=
edWeb2-svn20071201/twisted/web2/client/http.py
*** TwistedWeb2-svn20071201.orig/twisted/web2/client/http.py	Sat Dec  1 04:=
11:46 2007
--- TwistedWeb2-svn20071201/twisted/web2/client/http.py	Fri Mar 28 04:39:49=
 2008
***************
*** 8,13 ****
--- 8,14 ----
  =

  from zope.interface import implements
  =

+ from twisted.internet.error import ConnectionDone
  from twisted.internet.defer import Deferred
  from twisted.protocols.basic import LineReceiver
  from twisted.protocols.policies import TimeoutMixin
***************
*** 165,171 ****
          self._error(ProtocolError(text))
  =

      def connectionLost(self, reason):
!         self._error(reason)
  =

      def gotInitialLine(self, initialLine):
          parts =3D initialLine.split(' ', 2)
--- 166,177 ----
          self._error(ProtocolError(text))
  =

      def connectionLost(self, reason):
!         # If the response was non-chunked and of indeterminate length, tr=
eat the
!         # connection being closed cleanly as the end of the response
!         if self.length is None and not self.chunkedIn and reason.check(Co=
nnectionDone) and hasattr(self, 'stream'):
!             self.allContentReceived()
!         else:
!             self._error(reason)
  =

      def gotInitialLine(self, initialLine):
          parts =3D initialLine.split(' ', 2)