Ticket #5192: 5192_4.patch

File 5192_4.patch, 35.9 KB (added by darfire, 5 years ago)
  • twisted/web/_newclient.py

     
    573573        self.persistent = persistent
    574574
    575575
    576     def _writeHeaders(self, transport, TEorCL):
     576    def _writeHeaders(self, transport, bodyFramingHeader):
    577577        hosts = self.headers.getRawHeaders('host', ())
    578578        if len(hosts) != 1:
    579579            raise BadHeaders("Exactly one Host header required")
     
    586586            '%s %s HTTP/1.1\r\n' % (self.method, self.uri))
    587587        if not self.persistent:
    588588            requestLines.append('Connection: close\r\n')
    589         if TEorCL is not None:
    590             requestLines.append(TEorCL)
     589        if bodyFramingHeader is not None:
     590            requestLines.append(bodyFramingHeader)
    591591        for name, values in self.headers.getAllRawHeaders():
    592592            requestLines.extend(['%s: %s\r\n' % (name, v) for v in values])
    593593        requestLines.append('\r\n')
    594594        transport.writeSequence(requestLines)
    595595
    596 
    597     def _writeToChunked(self, transport):
     596    def _writeBodyToChunked(self, transport):
    598597        """
    599         Write this request to the given transport using chunked
     598        Write this request's body to the given transport using chunked
    600599        transfer-encoding to frame the body.
    601600        """
    602         self._writeHeaders(transport, 'Transfer-Encoding: chunked\r\n')
    603601        encoder = ChunkedEncoder(transport)
    604602        encoder.registerProducer(self.bodyProducer, True)
    605603        d = self.bodyProducer.startProducing(encoder)
     
    618616        return d
    619617
    620618
    621     def _writeToContentLength(self, transport):
     619    def _writeBodyToContentLength(self, transport):
    622620        """
    623         Write this request to the given transport using content-length to frame
    624         the body.
     621        Write this request's body to the given transport using content-length
     622        to frame the body.
    625623        """
    626         self._writeHeaders(
    627             transport,
    628             'Content-Length: %d\r\n' % (self.bodyProducer.length,))
    629624
    630625        # This Deferred is used to signal an error in the data written to the
    631626        # encoder below.  It can only errback and it will only do so before too
     
    737732            been completely written to the transport or with a L{Failure} if
    738733            there is any problem generating the request bytes.
    739734        """
     735        self._writeHeadersTo(transport)
     736        return self._writeBodyTo(transport)
     737
     738    def _writeHeadersTo(self, transport):
     739        """
     740        Format this L{Request}'s headers as HTTP/1.1 and write them
     741        to the given transport.
     742        """
     743        bodyFramingHeader = None
    740744        if self.bodyProducer is not None:
     745           if self.bodyProducer.length is UNKNOWN_LENGTH:
     746               bodyFramingHeader = 'Transfer-Encoding: chunked\r\n'
     747           else:
     748               bodyFramingHeader = 'Content-Length: %d\r\n' % (
     749                       self.bodyProducer.length,)
     750
     751        self._writeHeaders(transport, bodyFramingHeader)
     752
     753    def _writeBodyTo(self, transport):
     754        """
     755        Write this L{Request}'s body to the given transport, framing it
     756        according to the given headers('Transport-Encoding' or
     757        'Content-Length').
     758
     759        @return: A L{Deferred} which fires with C{None} when the request has
     760            been completely written the transport or with a L{Failure} if there
     761            is any problem generating the request body bytes. If bodyProducer
     762            is None the returned L{Deferred} is already fired.
     763        """
     764        if self.bodyProducer is not None:
    741765            if self.bodyProducer.length is UNKNOWN_LENGTH:
    742                 return self._writeToChunked(transport)
     766                return self._writeBodyToChunked(transport)
    743767            else:
    744                 return self._writeToContentLength(transport)
     768                return self._writeBodyToContentLength(transport)
    745769        else:
    746             self._writeHeaders(transport, None)
    747770            return succeed(None)
    748771
    749772
     
    11861209            self._producer.pauseProducing()
    11871210
    11881211
     1212class DiscardWithDeferred(Protocol):
     1213    """
     1214    A L{Protocol} that discards all received data and that fires a L{Deferred}
     1215    when all data has been received.
    11891216
     1217    @ivar finishedDeferred: L{Deferred} which fires with C{None} when all data
     1218        has been received and with L{Failure} on error.
     1219
     1220    """
     1221
     1222    def __init__(self):
     1223        self.finishedDeferred = Deferred()
     1224
     1225    def dataReceived(self, data):
     1226        pass
     1227
     1228    def connectionLost(self, reason):
     1229        if reason.type == ResponseDone:
     1230            self.finishedDeferred.callback(None)
     1231        else:
     1232            self.finishedDeferred.errback(reason)
     1233
     1234
     1235TIMEOUT_100_CONTINUE = 1
     1236
     1237
    11901238class HTTP11ClientProtocol(Protocol):
    11911239    """
    11921240    L{HTTP11ClientProtocol} is an implementation of the HTTP 1.1 client
    1193     protocol.  It supports as few features as possible.
     1241    protocol. It supports as few features as possible.
    11941242
    11951243    @ivar _parser: After a request is issued, the L{HTTPClientParser} to
    11961244        which received data making up the response to that request is
    11971245        delivered.
    11981246
     1247    @ivar _reactor: The reactor used for eventual callLater calls.
     1248
    11991249    @ivar _finishedRequest: After a request is issued, the L{Deferred} which
    12001250        will fire when a L{Response} object corresponding to that request is
    12011251        available.  This allows L{HTTP11ClientProtocol} to fail the request
     
    12161266        received.  This is eventually chained with C{_finishedRequest}, but
    12171267        only in certain cases to avoid double firing that Deferred.
    12181268
     1269    @ivar _responseBodyDeferred: After a request is issued, the L{Deferred}
     1270        that fires when the C{_parser} has done parsing the entire L{Response},
     1271        including body, with the data that came after the current L{Response}.
     1272        This can be used to set another C{_parser} but usually this decision
     1273        is done on C{_responseDeferred}'s callback.
     1274
     1275    @ivar _forcedRequestBody: True if we had an 100-Continue L{Request} whose
     1276        body was forcefully written to the transport because the server
     1277        did not respond in time with a L{Response} (possibly because of a
     1278        buggy server that doesn't implement expectations correctly).
     1279
     1280    @ivar _firstResponseTimer: A L{Delayed} that fires after
     1281        TIMEOUT_100_CONTINUE seconds and forcefully sends the L{Request} body
     1282        to the server.
     1283
     1284    @ivar _firstResponseDeferred: A L{Deferred} that fires with the first
     1285        L{Response} to an 100-Continue L{Request}. This may be an 100-Continue
     1286        response or a response with a final status. It may fire in the
     1287        WAITING_100_CONTINUE_RESPONSE, TRANSMITTING or WAITING states.
     1288
     1289    @ivar _firstResponseBodyDeferred: A L{Deferred} that fires when the body
     1290        of the first L{Response} to an 100-Continue L{Request} has been
     1291        successfully parsed. It signals that we can start sending the
     1292        L{Request} body. It fires in the WAITING_100_CONTINUE_RESPONSE_BODY,
     1293        TRANSMITTING and WAITING states.
     1294
    12191295    @ivar _state: Indicates what state this L{HTTP11ClientProtocol} instance
    12201296        is in with respect to transmission of a request and reception of a
    12211297        response.  This may be one of the following strings:
     
    12361312          - GENERATION_FAILED: There was an error while the request.  The
    12371313            request was not fully sent to the network.
    12381314
     1315          - WAITING_100_CONTINUE_RESPONSE: We're waiting for a L{Response} to a
     1316            L{Request} that expects 100-Continue.
     1317
     1318          - WAITING_100_CONTINUE_RESPONSE_BODY: Got an 100 Continue
     1319            L{Response} and we're discarding its body before sending the
     1320            L{Request}.
     1321
    12391322          - WAITING: The request was fully sent to the network.  The
    12401323            instance is now waiting for the response to be fully received.
    12411324
     
    12551338    _responseDeferred = None
    12561339
    12571340
    1258     def __init__(self, quiescentCallback=lambda c: None):
     1341    def __init__(self, quiescentCallback=lambda c: None, reactor=None):
     1342        """
     1343        Initialize this L{HTTP11ClientProtocol}. Optionally a reactor can be
     1344        given. Otherwise use the global reactor.
     1345        """
     1346        if reactor is None:
     1347            from twisted.internet import reactor
     1348        self._reactor = reactor
    12591349        self._quiescentCallback = quiescentCallback
    12601350        self._abortDeferreds = []
    12611351
     
    12841374            may errback with L{RequestNotSent} if it is not possible to send
    12851375            any more requests using this L{HTTP11ClientProtocol}.
    12861376        """
     1377
    12871378        if self._state != 'QUIESCENT':
    12881379            return fail(RequestNotSent())
    12891380
    1290         self._state = 'TRANSMITTING'
    1291         _requestDeferred = maybeDeferred(request.writeTo, self.transport)
     1381        if request.headers.hasHeader('expect'):
     1382            _expectations = request.headers.getRawHeaders('expect')
     1383            _expects100Continue = '100-continue' in [x.lower() for x in
     1384                                                     _expectations]
     1385        else:
     1386            _expects100Continue = False
     1387
    12921388        self._finishedRequest = Deferred()
    12931389
    1294         # Keep track of the Request object in case we need to call stopWriting
    1295         # on it.
    12961390        self._currentRequest = request
    12971391
    1298         self._transportProxy = TransportProxyProducer(self.transport)
    1299         self._parser = HTTPClientParser(request, self._finishResponse)
    1300         self._parser.makeConnection(self._transportProxy)
    1301         self._responseDeferred = self._parser._responseDeferred
     1392        if _expects100Continue:
     1393            self._handle100ContinueRequest(request)
     1394        else:
     1395            self._handleRequest(request)
    13021396
    13031397        def cbRequestWrotten(ignored):
    1304             if self._state == 'TRANSMITTING':
     1398            if self._state in ['TRANSMITTING',
     1399                               'WAITING_100_CONTINUE_RESPONSE',
     1400                               'WAITING_100_CONTINUE_RESPONSE_BODY']:
    13051401                self._state = 'WAITING'
    13061402                self._responseDeferred.chainDeferred(self._finishedRequest)
    13071403
     
    13151411                log.err(err, 'Error writing request, but not in valid state '
    13161412                             'to finalize request: %s' % self._state)
    13171413
    1318         _requestDeferred.addCallbacks(cbRequestWrotten, ebRequestWriting)
     1414        self._requestDeferred.addCallbacks(cbRequestWrotten, ebRequestWriting)
    13191415
     1416        self._responseBodyDeferred.addCallback(self._finishResponse)
     1417
    13201418        return self._finishedRequest
    13211419
    13221420
     
    13361434        # Currently the rest parameter is ignored. Don't forget to use it if
    13371435        # we ever add support for pipelining. And maybe check what trailers
    13381436        # mean.
    1339         if self._state == 'WAITING':
     1437        if self._state in ['WAITING',
     1438                'WAITING_100_CONTINUE_RESPONSE']:
    13401439            self._state = 'QUIESCENT'
    13411440        else:
    13421441            # The server sent the entire response before we could send the
     
    13691468                log.err()
    13701469                self.transport.loseConnection()
    13711470            self._disconnectParser(reason)
     1471            self._cleanupRequest()
    13721472
    13731473
    13741474    _finishResponse_TRANSMITTING = _finishResponse_WAITING
     1475    _finishResponse_WAITING_100_CONTINUE_RESPONSE = _finishResponse_WAITING
    13751476
     1477    def _handleRequest(self, request):
     1478        """
     1479        Send a non-100-Continue-expecting L{Request} to the transport.
    13761480
     1481        @param request: The L{Request} to be sent.
     1482
     1483        """
     1484        self._state = 'TRANSMITTING'
     1485        self._requestDeferred = maybeDeferred(request.writeTo, self.transport)
     1486        (self._responseDeferred,
     1487             self._responseBodyDeferred) = self._setupParser(request)
     1488
     1489
     1490    def _handle100ContinueRequest(self, request):
     1491        """
     1492        Send an 100-Continue L{Request} to the transport.
     1493
     1494        @param request: The L{Request} to be sent.
     1495        """
     1496        request._writeHeadersTo(self.transport)
     1497        self._state = 'WAITING_100_CONTINUE_RESPONSE'
     1498        self._forcedRequestBody = False
     1499        self._requestDeferred = Deferred()
     1500        self._responseDeferred = Deferred()
     1501        self._responseBodyDeferred = Deferred()
     1502        (self._firstResponseDeferred,
     1503             self._firstResponseBodyDeferred) = self._setupParser(request)
     1504        self._firstResponseTimer = self._reactor.callLater(
     1505            TIMEOUT_100_CONTINUE,
     1506            self._forceRequestBody)
     1507
     1508        self._firstResponseDeferred.addCallbacks(
     1509            self._handleFirstResponse, self._handle100ContinueError)
     1510
     1511
     1512    def _handleFirstResponse(self, response):
     1513        """
     1514        Handle the first L{Response} to an 100-Continue L{Request}. This may
     1515        be an 100-Continue or a final status L{Response}. If it is an 100
     1516        response, consume it's body and then send the L{Request} body, else
     1517        forward the response to the user by means of self._finishedRequest.
     1518
     1519        @param response: The L{Response} to the current L{Request}.
     1520        """
     1521        # This may be inactive if this is a response that came after the timer
     1522        # fired.
     1523        if self._firstResponseTimer.active():
     1524            self._firstResponseTimer.cancel()
     1525
     1526        if self._state == 'WAITING_100_CONTINUE_RESPONSE':
     1527
     1528            if response.code == 100:
     1529                self._state = 'WAITING_100_CONTINUE_RESPONSE_BODY'
     1530                self._discardResponseBody(response,
     1531                    self._handleFirstResponseBody,
     1532                    self._handle100ContinueError)
     1533            else:
     1534                # We're done with this request.
     1535                self._requestDeferred.callback(None)
     1536                self._forwardFirstResponse(response)
     1537
     1538        else:
     1539
     1540            if self._forcedRequestBody and response.code == 100:
     1541                # Late arrival, eat it.
     1542                self._discardResponseBody(response,
     1543                    self._handleFirstResponseBody,
     1544                    self._handle100ContinueError)
     1545
     1546            else:
     1547                # A late response that isn't 100-Continue; could be from a
     1548                # server that doesn't implement expectations correctly.
     1549                self._forcedRequestBody = False
     1550                self._forwardFirstResponse(response)
     1551
     1552
     1553    def _forwardFirstResponse(self, response):
     1554        """
     1555        Forward a L{Response} and its body to the caller.
     1556
     1557        @param response: The L{Response} to be forwarded.
     1558        """
     1559
     1560        # Save this since it may get cleaned up in _finishResponse
     1561        responseDeferred = self._responseDeferred
     1562        self._firstResponseBodyDeferred.chainDeferred(
     1563                self._responseBodyDeferred)
     1564        responseDeferred.callback(response)
     1565
     1566
     1567    def _handleFirstResponseBody(self, rest):
     1568        """
     1569        The body of the first L{Response} to the current 100-Continue
     1570        L{Request} has been parsed. If the L{Response} wasn't an 100-Continue
     1571        forward to self._responseBodyDeferred. Otherwise create a new parser
     1572        for the second L{Response}.
     1573
     1574        @param rest: Data that wasn't parsed by the parser because it came
     1575            after the L{Response}. If we reload the parser, initialize it
     1576            with this data.
     1577        """
     1578        # We've just done discarding an 100-Continue response's body. Might
     1579        # be because we're waiting to send the request body or it might be
     1580        # that we've ignored a late response.
     1581
     1582        self._forcedRequestBody = False
     1583
     1584        if self._state == 'WAITING_100_CONTINUE_RESPONSE_BODY':
     1585            self._state = 'TRANSMITTING'
     1586
     1587            # Send the request body.
     1588
     1589            _requestBodyDeferred = maybeDeferred(
     1590                self._currentRequest._writeBodyTo,
     1591                self.transport)
     1592
     1593            _requestBodyDeferred.chainDeferred(self._requestDeferred)
     1594
     1595        # In both cases create a new parser.
     1596
     1597        self._disconnectParser(None)
     1598
     1599        (_secondResponseDeferred, _secondResponseBodyDeferred) =\
     1600            self._setupParser(self._currentRequest, data=rest)
     1601
     1602        _secondResponseDeferred.chainDeferred(self._responseDeferred)
     1603        _secondResponseBodyDeferred.chainDeferred(self._responseBodyDeferred)
     1604
     1605
     1606    def _discardResponseBody(self, response, callback, errback):
     1607        """
     1608        Discard a L{Response}'s body and call callback when done and errback
     1609        on error.
     1610
     1611        @param response: L{Response} that needs to be discarded.
     1612        @param callback: function to be called when done.
     1613        @param errback: function to be called on error
     1614        """
     1615        discarder = DiscardWithDeferred()
     1616
     1617        # We use discarder.finishedDeferred to catch body parsing
     1618        # errors and self._firstResponseBodyDeferred to catch success.
     1619
     1620        discarder.finishedDeferred.addErrback(errback)
     1621
     1622        response.deliverBody(discarder)
     1623
     1624        self._firstResponseBodyDeferred.addCallback(callback)
     1625
     1626
     1627    def _forceRequestBody(self):
     1628        """
     1629        Send the current L{Request} body even though we were expecting an
     1630        100 or final status L{Response}. It may just be a broken server that
     1631        doesn't implement correctly expectations.
     1632        """
     1633        self._state = 'TRANSMITTING'
     1634
     1635        self._forcedRequestBody = True
     1636
     1637        _requestBodyDeferred = maybeDeferred(self._currentRequest._writeBodyTo,
     1638            self.transport)
     1639
     1640        _requestBodyDeferred.chainDeferred(self._requestDeferred)
     1641
     1642
     1643    def _setupParser(self, request, data=''):
     1644        """
     1645        Setup a L{HTTPClientParser} for a L{Response} to a L{Request}. If this
     1646        is not the first parser associated with this protocol, call
     1647        L{HTTP11ClientProtocol._disconnectParser} first. Pass the given C{data}
     1648        to the newly created parser.
     1649
     1650        @param request: L{Request} waiting for a L{Response}.
     1651        @param data: Data to initialize the L{HTTPClientParser} with.
     1652        """
     1653        self._transportProxy = TransportProxyProducer(self.transport)
     1654
     1655        _responseBodyDeferred = Deferred()
     1656
     1657        def cbOnBodyFinish(rest):
     1658            _responseBodyDeferred.callback(rest)
     1659
     1660        self._parser = HTTPClientParser(request, cbOnBodyFinish)
     1661
     1662        self._parser.makeConnection(self._transportProxy)
     1663
     1664        # Grab this before passing data, since it might disappear if data is a
     1665        # complete response.
     1666
     1667        _responseDeferred = self._parser._responseDeferred
     1668
     1669        self._parser.dataReceived(data)
     1670
     1671        return (_responseDeferred, _responseBodyDeferred)
     1672
     1673
     1674    def _cleanupOn100ContinueError(self):
     1675        """
     1676        State-dependent cleanup on parsing errors while handling an
     1677        100-Continue-expecting L{Request}.
     1678        """
     1679    _cleanupOn100ContinueError = makeStatefulDispatcher(
     1680            'cleanupOn100ContinueError', _cleanupOn100ContinueError)
     1681
     1682
     1683    def _cleanupOn100ContinueError_WAITING_100_CONTINUE_RESPONSE(self):
     1684        """
     1685        The L{Request} body no sent yet. Fire self._requestDeferred because
     1686        we've basically finished dealing with this L{Request}. Also, this
     1687        forwards the L{Failure} to the user.
     1688        """
     1689        self._requestDeferred.callback(None)
     1690
     1691
     1692    def _cleanupOn100ContinueError_WAITING_100_CONTINUE_RESPONSE_BODY(self):
     1693        """
     1694        The L{Request} body no sent yet. Fire self._requestDeferred because
     1695        we've basically finished dealing with this L{Request}. Also, this
     1696        forwards the L{Failure} to the user.
     1697        """
     1698        self._requestDeferred.callback(None)
     1699
     1700
     1701    def _cleanupOn100ContinueError_TRANSMITTING(self):
     1702        """
     1703        We're currently sending the L{Request} body. The error will be sent to
     1704        the user after the body has been sent. No cleanup needed.
     1705        """
     1706
     1707
     1708    def _cleanupOn100ContinueError_WAITING(self):
     1709        """
     1710        No cleanup needed.
     1711        """
     1712
     1713
     1714    def _handle100ContinueError(self, err):
     1715        """
     1716        Handle any L{Failure} that could occur while handling a L{Request} that
     1717        expects 100-Continue. These are errors on parsing the first response
     1718        and the first response's body and can occur in the
     1719        WAITING_100_CONTINUE_RESPONSE/WAITING_100_CONTINUE_RESPONSE_BODY if
     1720        the server supports expectations or TRANSMITTING/WAITING if the
     1721        L{Request} body was sent after TIMEOUT_100_CONTINUE. Depending on
     1722        the current state some cleanup needs to be performed and the L{Failure}
     1723        is forwarded to self._responseDeferred.
     1724
     1725        @param err: L{Failure} to be forwarded.
     1726        """
     1727        self._cleanupOn100ContinueError()
     1728        self._responseDeferred.errback(err)
     1729
     1730
     1731    def _cleanupRequest(self):
     1732        """
     1733        Clean up the L{HTTP11ClientProtocol}'s state after a L{Request} has
     1734        been done with.
     1735        """
     1736        self._currentRequest = None
     1737        self._finishedRequest = None
     1738        self._responseDeferred = None
     1739        self._firstResponseDeferred = None
     1740        self._firstResponseBodyDeferred = None
     1741
     1742
    13771743    def _disconnectParser(self, reason):
    13781744        """
    13791745        If there is still a parser, call its C{connectionLost} method with the
     
    13841750        if self._parser is not None:
    13851751            parser = self._parser
    13861752            self._parser = None
    1387             self._currentRequest = None
    1388             self._finishedRequest = None
    1389             self._responseDeferred = None
    13901753
    13911754            # The parser is no longer allowed to do anything to the real
    13921755            # transport.  Stop proxying from the parser's transport to the real
     
    14881851        self._abortDeferreds = []
    14891852
    14901853
     1854    def _connectionLost_WAITING_100_CONTINUE_RESPONSE(self, reason):
     1855        """
     1856        Disconnect the parser so that it can propagate the event and move to
     1857        the C{'CONNECTION_LOST'} state.
     1858        """
     1859        self._disconnectParser(reason)
     1860        self._state = 'CONNECTION_LOST'
     1861
     1862
     1863    def _connectionLost_WAITING_100_CONTINUE_RESPONSE_BODY(self, reason):
     1864        """
     1865        Disconnect the parser so that it can propagate the event and move to
     1866        the C{'CONNECTION_LOST'} state.
     1867        """
     1868        self._disconnectParser(reason)
     1869        self._state = 'CONNECTION_LOST'
     1870
     1871
    14911872    def abort(self):
    14921873        """
    14931874        Close the connection and cause all outstanding L{request} L{Deferred}s
  • twisted/web/test/test_newclient.py

     
    77
    88__metaclass__ = type
    99
     10from StringIO import StringIO
     11
    1012from zope.interface import implements
    1113from zope.interface.verify import verifyObject
    1214
     
    1618from twisted.internet.error import ConnectionDone, ConnectionLost
    1719from twisted.internet.defer import Deferred, succeed, fail
    1820from twisted.internet.protocol import Protocol
     21from twisted.internet.task import Clock, Cooperator
    1922from twisted.trial.unittest import TestCase
    2023from twisted.test.proto_helpers import StringTransport, AccumulatingProtocol
    2124from twisted.web._newclient import UNKNOWN_LENGTH, STATUS, HEADER, BODY, DONE
     
    2730from twisted.web._newclient import ConnectionAborted, ResponseNeverReceived
    2831from twisted.web._newclient import BadHeaders, ResponseDone, PotentialDataLoss, ExcessWrite
    2932from twisted.web._newclient import TransportProxyProducer, LengthEnforcingConsumer, makeStatefulDispatcher
     33from twisted.web._newclient import TIMEOUT_100_CONTINUE
    3034from twisted.web.http_headers import Headers
    3135from twisted.web.http import _DataLoss
    3236from twisted.web.iweb import IBodyProducer, IResponse
     37from twisted.web.client import FileBodyProducer
    3338
    3439
    3540
     
    827832    method = 'GET'
    828833    stopped = False
    829834    persistent = False
     835    headers = Headers()
    830836
    831837    def writeTo(self, transport):
    832838        self.finished = Deferred()
     
    846852    L{Request} with no body producer.
    847853    """
    848854    persistent = False
     855    headers = Headers()
    849856
    850857    def writeTo(self, transport):
    851858        transport.write('SOME BYTES')
     
    863870        Create an L{HTTP11ClientProtocol} connected to a fake transport.
    864871        """
    865872        self.transport = StringTransport()
    866         self.protocol = HTTP11ClientProtocol()
     873        self.clock = Clock()
     874        self.protocol = HTTP11ClientProtocol(reactor=self.clock)
    867875        self.protocol.makeConnection(self.transport)
    868876
    869877
     
    916924        """
    917925        class BrokenRequest:
    918926            persistent = False
     927            headers = Headers()
    919928            def writeTo(self, transport):
    920929                return fail(ArbitraryException())
    921930
     
    939948        """
    940949        class BrokenRequest:
    941950            persistent = False
     951            headers = Headers()
    942952            def writeTo(self, transport):
    943953                raise ArbitraryException()
    944954
     
    15661576        self.assertTrue(transport.disconnecting)
    15671577
    15681578
     1579    def _send100ContinueRequest(self, body, persistent=False):
     1580        """
     1581        Send a L{Request} that expects 100-Continue with the given body.
     1582        """
     1583        def _immediateScheduler(x):
     1584            return succeed(x())
    15691585
     1586        cooperator = Cooperator(scheduler=_immediateScheduler, started=False)
     1587        producer = FileBodyProducer(StringIO(body), cooperator=cooperator)
     1588
     1589        headers = Headers({'host': ['example.com'], 'expect': ['100-Continue']})
     1590
     1591        d = self.protocol.request(Request('POST', '/foo', headers, producer,
     1592            persistent=persistent))
     1593
     1594        self.transport.clear()
     1595
     1596        cooperator.start()
     1597
     1598        self.assertEqual(self.transport.value(), '')
     1599
     1600        return d
     1601
     1602
     1603    def test_expect100ContinueGetFinalStatus(self):
     1604        """
     1605        When we expect 100-Continue and get a final status L{Response} we don't
     1606        send the L{Request} body and return the first L{Response} to the user.
     1607        """
     1608        d = self._send100ContinueRequest('x' * 10)
     1609
     1610        def cbResponse(response):
     1611            self.assertEqual(response.code, 200)
     1612
     1613        d.addCallback(cbResponse)
     1614
     1615        self.protocol.dataReceived(
     1616            "HTTP/1.1 200 OK\r\n"
     1617            "Content-length: 0\r\n"
     1618            "\r\n")
     1619
     1620        self.assertEqual(self.transport.value(), '')
     1621
     1622        return d
     1623
     1624
     1625    def test_expect100ContinueGet100Continue(self):
     1626        """
     1627        When we expect 100-Continue and get an 100-Continue L{Response} we send
     1628        the L{Request} body and return the second L{Response} to the user.
     1629        """
     1630        d = self._send100ContinueRequest('x' * 10)
     1631
     1632        def cbResponse(response):
     1633            self.assertEqual(response.code, 200)
     1634
     1635        d.addCallback(cbResponse)
     1636
     1637        self.protocol.dataReceived(
     1638            "HTTP/1.1 100 Continue\r\n"
     1639            "Content-Length: 3\r\n"
     1640            "\r\n"
     1641            "123")
     1642
     1643        self.protocol.dataReceived(
     1644            "HTTP/1.1 200 OK\r\n"
     1645            "Content-Length: 0\r\n"
     1646            "\r\n")
     1647
     1648        self.assertEqual(self.transport.value(), 'x' * 10)
     1649
     1650        return d
     1651
     1652
     1653    def test_expect100ContinueGet100ContinueBackToBack(self):
     1654        """
     1655        When we expect 100-Continue and we get 2 response back to back (100 and
     1656        final status) we should act as if they came separately.
     1657        """
     1658        d = self._send100ContinueRequest('x' * 10)
     1659
     1660        def cbResponse(response):
     1661            self.assertEqual(response.code, 200)
     1662
     1663        d.addCallback(cbResponse)
     1664
     1665        self.protocol.dataReceived(
     1666            "HTTP/1.1 100 Continue\r\n"
     1667            "Content-Length: 3\r\n"
     1668            "\r\n"
     1669            "123"
     1670            "HTTP/1.1 200 OK\r\n"
     1671            "Content-Length: 0\r\n"
     1672            "\r\n")
     1673
     1674        self.assertEqual(self.transport.value(), 'x' * 10)
     1675
     1676        return d
     1677
     1678
     1679    def test_expect100ContinueServerBroken(self):
     1680        """
     1681        When we expect 100-Continue and the server is broken and waits for the
     1682        L{Request} body we wait for a limited amount and then send the body.
     1683        """
     1684        d = self._send100ContinueRequest('x' * 10)
     1685
     1686        def cbResponse(response):
     1687            self.assertEqual(response.code, 200)
     1688
     1689        d.addCallback(cbResponse)
     1690
     1691        self.clock.advance(TIMEOUT_100_CONTINUE + 1)
     1692
     1693        self.assertEqual(self.transport.value(), 'x' * 10)
     1694
     1695        self.protocol.dataReceived(
     1696            "HTTP/1.1 200 OK\r\n"
     1697            "Content-Length: 0\r\n"
     1698            "\r\n")
     1699
     1700        return d
     1701
     1702
     1703    def test_expect100ContinueTimerFiresLate100ContinueResponse(self):
     1704        """
     1705        When we expect 100-Continue and the server is slow and sends an
     1706        100-Continue after we sent the body we consume the 100-Continue
     1707        L{Response} and return the second L{Response} to the user.
     1708        """
     1709        d = self._send100ContinueRequest('x' * 10)
     1710
     1711        def cbResponse(response):
     1712            self.assertEqual(response.code, 200)
     1713
     1714        d.addCallback(cbResponse)
     1715
     1716        self.clock.advance(TIMEOUT_100_CONTINUE + 1)
     1717
     1718        self.assertEqual(self.transport.value(), 'x' * 10)
     1719
     1720        self.protocol.dataReceived(
     1721            "HTTP/1.1 100 Continue\r\n"
     1722            "Content-length: 3\r\n"
     1723            "\r\n"
     1724            "123")
     1725
     1726        self.protocol.dataReceived(
     1727            "HTTP/1.1 200 OK\r\n"
     1728            "Content-length: 0\r\n"
     1729            "\r\n")
     1730
     1731        return d
     1732
     1733
     1734    _garbageResponse = "unparseable garbage goes here\r\n"
     1735
     1736
     1737    def test_expect100ContinueBrokenFirstResponse(self):
     1738        """
     1739        When we expect 100-Continue and the first L{Response} is broken, return
     1740        the error to the user.
     1741        """
     1742        d = self._send100ContinueRequest('x' * 10)
     1743
     1744        self.protocol.dataReceived(self._garbageResponse)
     1745
     1746        self.assertEqual(self.transport.value(), '')
     1747
     1748        return assertResponseFailed(self, d, [ParseError])
     1749
     1750
     1751    def test_expect100ContinueBrokenFirstResponseChunkedBody(self):
     1752        """
     1753        When we expect 100-Continue and the 100-Continue L{Response} has a
     1754        chunked body and it is broken, return the error to the user.
     1755        """
     1756        d = self._send100ContinueRequest('x' * 10)
     1757
     1758        self.protocol.dataReceived(
     1759            "HTTP/1.1 100 Continue\r\n"
     1760            "Transfer-Encoding: chunked\r\n"
     1761            "\r\n")
     1762
     1763        self.protocol.dataReceived("3\r\nzzz\r\n")
     1764        self.protocol.dataReceived("3\r\nzzz\r\nzzz\r\n") #incorrect chunk
     1765
     1766        self.assertEqual(self.transport.value(), '')
     1767
     1768        return assertResponseFailed(self, d, [ValueError, _DataLoss])
     1769
     1770
     1771    def test_expect100ContinueBrokenSecondResponse(self):
     1772        """
     1773        When we expect 100-Continue and the 100-Continue L{Response} is ok but
     1774        the second L{Response} is broken, return the error to the user.
     1775        """
     1776        d = self._send100ContinueRequest('x' * 10)
     1777
     1778        self.protocol.dataReceived(
     1779            "HTTP/1.1 100 Continue\r\n"
     1780            "Content-length: 3\r\n"
     1781            "\r\n"
     1782            "123")
     1783
     1784        self.protocol.dataReceived(self._garbageResponse)
     1785
     1786        self.assertEqual(self.transport.value(), 'x' * 10)
     1787
     1788        return assertResponseFailed(self, d, [ParseError])
     1789
     1790
     1791    def _setupForQuiescent(self):
     1792        self.quiescentResult = []
     1793
     1794        def callback(p):
     1795            self.assertEqual(p, self.protocol)
     1796            self.assertEqual(p.state, "QUIESCENT")
     1797            self.quiescentResult.append(p)
     1798
     1799        self.transport = StringTransport()
     1800        self.clock = Clock()
     1801        self.protocol = HTTP11ClientProtocol(callback, reactor=self.clock)
     1802        self.protocol.makeConnection(self.transport)
     1803
     1804
     1805    def _checkQuiescentCalled(self, requestDeferred, body=''):
     1806        # Headers done, but still no quiescent callback:
     1807        self.assertEqual(self.quiescentResult, [])
     1808
     1809        result = []
     1810        requestDeferred.addCallback(result.append)
     1811        response = result[0]
     1812
     1813        # When response body is done (i.e. connectionLost is called), note the
     1814        # fact in quiescentResult:
     1815        bodyProtocol = AccumulatingProtocol()
     1816        bodyProtocol.closedDeferred = Deferred()
     1817        bodyProtocol.closedDeferred.addCallback(
     1818            lambda ign: self.quiescentResult.append("response done"))
     1819
     1820        response.deliverBody(bodyProtocol)
     1821        self.protocol.dataReceived(body)
     1822        bodyProtocol.closedReason.trap(ResponseDone)
     1823        # Quiescent callback called *before* self.protocol handling the response
     1824        # body gets its connectionLost called:
     1825        self.assertEqual(self.quiescentResult, [self.protocol, "response done"])
     1826
     1827        # Make sure everything was cleaned up:
     1828        self.assertEqual(self.protocol._parser, None)
     1829        self.assertEqual(self.protocol._finishedRequest, None)
     1830        self.assertEqual(self.protocol._currentRequest, None)
     1831        self.assertEqual(self.protocol._transportProxy, None)
     1832        self.assertEqual(self.protocol._responseDeferred, None)
     1833
     1834
     1835    def test_expect100ContinueGot100ContinueQuiescentCallbackCalled(self):
     1836        """
     1837        We have a persistent connection and we expect 100-Continue. When we
     1838        get an 100-Continue L{Response} the quiescent callback needs to be
     1839        called.
     1840        """
     1841        self._setupForQuiescent()
     1842
     1843        d = self._send100ContinueRequest('x' * 10, persistent=True)
     1844
     1845        def cbResponse(response):
     1846            self.assertEqual(response.code, 200)
     1847            return response
     1848
     1849        d.addCallback(cbResponse)
     1850
     1851        self.protocol.dataReceived(
     1852            "HTTP/1.1 100 Continue\r\n"
     1853            "Content-Length: 3\r\n"
     1854            "\r\n"
     1855            "123")
     1856
     1857        self.protocol.dataReceived(
     1858            "HTTP/1.1 200 OK\r\n"
     1859            "Content-Length: 3\r\n"
     1860            "\r\n")
     1861
     1862        self._checkQuiescentCalled(d, body="abc")
     1863
     1864        self.assertEqual(self.transport.value(), 'x' * 10)
     1865
     1866        return d
     1867
     1868
     1869    def test_expect100ContinueGotFinalStatusQuiescentCallbackCalled(self):
     1870        """
     1871        We have a persistent connection and we expect 100-Continue. When we
     1872        get a L{Response} with a final status the quiescent callback needs to
     1873        be called.
     1874        """
     1875        self._setupForQuiescent()
     1876
     1877        d = self._send100ContinueRequest('x' * 10, persistent=True)
     1878
     1879        def cbResponse(response):
     1880            self.assertEqual(response.code, 200)
     1881            return response
     1882
     1883        d.addCallback(cbResponse)
     1884
     1885        self.protocol.dataReceived(
     1886            "HTTP/1.1 200 OK\r\n"
     1887            "Content-Length: 3\r\n"
     1888            "\r\n")
     1889
     1890        self._checkQuiescentCalled(d, body='abc')
     1891
     1892        self.assertEqual(self.transport.value(), '')
     1893
     1894        return d
     1895
     1896   
     1897    def test_expect100ContinueConnectionLostWhileWaitingFirstResponse(self):
     1898        """
     1899        When we expect 100-Continue and we get disconnected while waiting for
     1900        the first L{Response}, the L{Deferred} return by request() must errback
     1901        a ResponseFailed wrapping the underlying failure.
     1902        """
     1903        d = self._send100ContinueRequest('x' * 10)
     1904
     1905        self.assertEqual(self.transport.value(), '')
     1906
     1907        self.protocol.connectionLost(Failure(ArbitraryException()))
     1908
     1909        self.assertEqual(self.protocol._state, 'CONNECTION_LOST')
     1910
     1911        return assertResponseFailed(
     1912            self, d, [ArbitraryException])
     1913
     1914
     1915    def test_expect100ContinueConnectionLostWhileWaitingFirstResponseBody(self):
     1916        """
     1917        When we expect 100-Continue and we get an 100-Continue L{Response}
     1918        and we get disconnected while waiting for its body, the L{Deferred}
     1919        returned by request() must errback a ResponseFailed wrapping the
     1920        underlying failure.
     1921        """
     1922        d = self._send100ContinueRequest('x' * 10)
     1923
     1924        self.assertEqual(self.transport.value(), '')
     1925
     1926        self.protocol.dataReceived(
     1927            "HTTP/1.1 100 Continue\r\n"
     1928            "Content-length: 10\r\n"
     1929            "\r\n")
     1930
     1931        self.protocol.connectionLost(Failure(ArbitraryException()))
     1932
     1933        self.assertEqual(self.protocol._state, 'CONNECTION_LOST')
     1934
     1935        return assertResponseFailed(
     1936            self, d, [ArbitraryException, _DataLoss])
     1937
     1938
    15701939class StringProducer:
    15711940    """
    15721941    L{StringProducer} is a dummy body producer.