Ticket #5192: 5192_2.patch

File 5192_2.patch, 15.8 KB (added by darfire, 5 years ago)
  • twisted/web/test/test_newclient.py

     
    1616from twisted.internet.error import ConnectionDone
    1717from twisted.internet.defer import Deferred, succeed, fail
    1818from twisted.internet.protocol import Protocol
     19from twisted.internet import reactor
    1920from twisted.trial.unittest import TestCase
    2021from twisted.test.proto_helpers import StringTransport, AccumulatingProtocol
    2122from twisted.web._newclient import UNKNOWN_LENGTH, STATUS, HEADER, BODY, DONE
     
    2728from twisted.web._newclient import ConnectionAborted
    2829from twisted.web._newclient import BadHeaders, ResponseDone, PotentialDataLoss, ExcessWrite
    2930from twisted.web._newclient import TransportProxyProducer, LengthEnforcingConsumer, makeStatefulDispatcher
     31from twisted.web._newclient import TIMEOUT_100_CONTINUE
    3032from twisted.web.http_headers import Headers
    3133from twisted.web.http import _DataLoss
    3234from twisted.web.iweb import IBodyProducer, IResponse
     
    793795    """
    794796    method = 'GET'
    795797    stopped = False
     798    headers = Headers()
    796799
    797800    def writeTo(self, transport):
    798801        self.finished = Deferred()
     
    811814    returns a succeeded L{Deferred}.  This vaguely emulates the behavior of a
    812815    L{Request} with no body producer.
    813816    """
     817    headers = Headers()
    814818    def writeTo(self, transport):
    815819        transport.write('SOME BYTES')
    816820        return succeed(None)
    817821
    818822
     823class BufferProducer(object):
     824    """
     825    A body producer that outputs a string with a known length.
     826    """
     827    implements(IBodyProducer)
    819828
     829    def __init__(self, data):
     830        self.data = data
     831        self.length = len(data)
     832
     833    def startProducing(self, consumer):
     834        consumer.write(self.data)
     835        return succeed(None)
     836
     837    def pauseProducing(self):
     838        pass
     839
     840    def stopProducing(self):
     841        pass
     842
    820843class HTTP11ClientProtocolTests(TestCase):
    821844    """
    822845    Tests for the HTTP 1.1 client protocol implementation,
     
    879902        L{RequestGenerationFailed} wrapping the underlying failure.
    880903        """
    881904        class BrokenRequest:
     905            headers = Headers()
    882906            def writeTo(self, transport):
    883907                return fail(ArbitraryException())
    884908
     
    901925        a L{Failure} of L{RequestGenerationFailed} wrapping that exception.
    902926        """
    903927        class BrokenRequest:
     928            headers = Headers()
    904929            def writeTo(self, transport):
    905930                raise ArbitraryException()
    906931
     
    13121337                                        [ConnectionAborted, _DataLoss])
    13131338        return deferred.addCallback(checkError)
    13141339
     1340    def test_expect100ContinueGot100Continue(self):
     1341        """
     1342        When we expect 100-Continue and we receive an 100-Continue L{Response}
     1343        the user writes the L{Request} body and receives back the second(final)
     1344        L{Response}.
     1345        """
     1346        producer = BufferProducer('X' * 10)
    13151347
     1348        headers = Headers({'host': ['example.com'],
     1349            'expect': ['100-Continue']})
    13161350
     1351        d = self.protocol.request(Request('POST', '/foo', headers, producer))
     1352
     1353        self.transport.clear()
     1354
     1355        def cb100Continue(response):
     1356            self.assertEqual(response.code, 200)
     1357
     1358        d.addCallback(cb100Continue)
     1359
     1360        self.protocol.dataReceived(
     1361                "HTTP/1.1 100 Continue\r\n"
     1362                "Content-Length: 3\r\n"
     1363                "\r\n"
     1364                '123'
     1365                )
     1366
     1367        self.protocol.dataReceived(
     1368                "HTTP/1.1 200 OK\r\n"
     1369                "Content-Length: 0\r\n"
     1370                "\r\n"
     1371                )
     1372
     1373        self.assertEqual(self.transport.value(), 'X' * 10)
     1374
     1375        return d
     1376
     1377
     1378    def test_expect100ContinueGotFinalStatus(self):
     1379        """
     1380        When we expect 100-Continue and we receive a final status L{Response}
     1381        we don't write the L{Request} body anymore and the user receives the
     1382        first L{Response}.
     1383        """
     1384        producer = BufferProducer('X' * 10)
     1385
     1386        headers = Headers({'host': ['example.com'], 'expect': ['100-Continue']})
     1387
     1388        d = self.protocol.request(Request('POST', '/foo', headers, producer))
     1389
     1390        self.transport.clear()
     1391
     1392        def cb100Continue(response):
     1393            self.assertEqual(response.code, 200)
     1394
     1395        d.addCallback(cb100Continue)
     1396
     1397
     1398        self.protocol.dataReceived(
     1399                "HTTP/1.1 200 OK\r\n"
     1400                "Content-Length: 0\r\n"
     1401                "\r\n"
     1402                )
     1403
     1404        self.assertEqual(self.transport.value(), '')
     1405
     1406        return d
     1407
     1408    def test_expect100ContinueServerNotSupporting(self):
     1409        """
     1410        When we expect 100-continue and the server doesn't respond with a
     1411        response for upto TIMEOUT_100_CONTINUE seconds (maybe because it does
     1412        not correctly handle this expectation) we continue with sending the
     1413        request body.
     1414        """
     1415        producer = BufferProducer('X' * 10)
     1416
     1417        headers = Headers({'host': ['example.com'], 'expect': ['100-Continue']})
     1418
     1419        d = self.protocol.request(Request('POST', '/foo', headers, producer))
     1420
     1421        self.transport.clear()
     1422
     1423        def cbResponse(response):
     1424            self.assertEqual(response.code, 200)
     1425
     1426        d.addCallback(cbResponse)
     1427
     1428        d2 = Deferred()
     1429
     1430        d.chainDeferred(d2)
     1431
     1432        def laterCall():
     1433            try:
     1434                self.assertEqual(self.transport.value(), 'X'*10)
     1435            except Exception, e:
     1436                d2.errback(e)
     1437                return
     1438
     1439            self.protocol.dataReceived(
     1440                    "HTTP/1.1 200 OK\r\n"
     1441                    "Content-Length: 0\r\n"
     1442                    "\r\n"
     1443                    )
     1444
     1445        reactor.callLater(TIMEOUT_100_CONTINUE + 1.0, laterCall)
     1446
     1447        self.assertEqual(self.transport.value(), '')
     1448
     1449        return d2
     1450
     1451
     1452
    13171453class StringProducer:
    13181454    """
    13191455    L{StringProducer} is a dummy body producer.
  • twisted/web/_newclient.py

     
    3737from twisted.internet.interfaces import IConsumer, IPushProducer
    3838from twisted.internet.error import ConnectionDone
    3939from twisted.internet.defer import Deferred, succeed, fail, maybeDeferred
     40from twisted.internet import reactor
    4041from twisted.internet.protocol import Protocol
    4142from twisted.protocols.basic import LineReceiver
     43from twisted.protocols.wire import Discard
    4244from twisted.web.iweb import UNKNOWN_LENGTH, IResponse
    4345from twisted.web.http_headers import Headers
    4446from twisted.web.http import NO_CONTENT, NOT_MODIFIED
     
    5153BODY = 'BODY'
    5254DONE = 'DONE'
    5355
     56# Interval to wait for a Response to a Request with 'Expect: 100-Continue'
     57TIMEOUT_100_CONTINUE = 1.0
    5458
     59
    5560class BadHeaders(Exception):
    5661    """
    5762    Headers passed to L{Request} were in some way invalid.
     
    516521            self._responseDeferred.errback(Failure(ResponseFailed([reason])))
    517522            del self._responseDeferred
    518523
     524    def _hasResponse(self):
     525        return hasattr(self, 'response')
    519526
    520527
     528
    521529class Request:
    522530    """
    523531    A L{Request} instance describes an HTTP request to be sent to an HTTP
     
    566574        transport.writeSequence(requestLines)
    567575
    568576
    569     def _writeToChunked(self, transport):
     577    def _writeBodyToChunked(self, transport):
    570578        """
    571         Write this request to the given transport using chunked
     579        Write this request's body to the given transport using chunked
    572580        transfer-encoding to frame the body.
    573581        """
    574         self._writeHeaders(transport, 'Transfer-Encoding: chunked\r\n')
    575582        encoder = ChunkedEncoder(transport)
    576583        encoder.registerProducer(self.bodyProducer, True)
    577584        d = self.bodyProducer.startProducing(encoder)
     
    590597        return d
    591598
    592599
    593     def _writeToContentLength(self, transport):
     600    def _writeBodyToContentLength(self, transport):
    594601        """
    595         Write this request to the given transport using content-length to frame
    596         the body.
     602        Write this request's body to the given transport using content-length
     603        to frame the body.
    597604        """
    598         self._writeHeaders(
    599             transport,
    600             'Content-Length: %d\r\n' % (self.bodyProducer.length,))
    601605
    602606        # This Deferred is used to signal an error in the data written to the
    603607        # encoder below.  It can only errback and it will only do so before too
     
    709713            been completely written to the transport or with a L{Failure} if
    710714            there is any problem generating the request bytes.
    711715        """
     716
     717        self._writeHeadersTo(transport)
     718        return self._writeBodyTo(transport)
     719
     720    def _writeHeadersTo(self, transport):
     721        """
     722        Format this L{Request}'s headers as HTTP/1.1 and write them
     723        synchronously to the given transport
     724        """
     725        TEorCL = None
    712726        if self.bodyProducer is not None:
    713727            if self.bodyProducer.length is UNKNOWN_LENGTH:
    714                 return self._writeToChunked(transport)
     728                TEorCL = "Transfer-Encoding: chunked\r\n"
    715729            else:
    716                 return self._writeToContentLength(transport)
     730                TEorCL = 'Content-Length: %d\r\n' % (self.bodyProducer.length,)
     731
     732        self._writeHeaders(transport, TEorCL)
     733
     734    def _writeBodyTo(self, transport):
     735        """
     736        Write this L{Request}'s body to the given transport, framing it
     737        according to the given headers('Transport-Encoding' or
     738        'Content-Length').
     739
     740        @return: A L{Deferred} which fires with C{None} when the request has
     741            been completely written the transport or with a L{Failure} if there
     742            is any problem generating the request body bytes. If bodyProducer
     743            is None the returned L{Deferred} is already fired.
     744        """
     745        if self.bodyProducer is not None:
     746            if self.bodyProducer.length is UNKNOWN_LENGTH:
     747                return self._writeBodyToChunked(transport)
     748            else:
     749                return self._writeBodyToContentLength(transport)
    717750        else:
    718             self._writeHeaders(transport, None)
    719751            return succeed(None)
    720752
    721 
    722753    def stopWriting(self):
    723754        """
    724755        Stop writing this request to the transport.  This can only be called
     
    11581189            self._producer.pauseProducing()
    11591190
    11601191
     1192class DiscardWithDeferred(Protocol):
     1193    """
     1194    A L{Protocol} that discards all received data and that fires a L{Deferred}
     1195    when all data has been received.
    11611196
     1197    @ivar finishedDeferred: L{Deferred} which fires with C{None} when all data
     1198        has been received and with L{Failure} on error.
     1199
     1200    """
     1201
     1202    def __init__(self):
     1203        self.finishedDeferred = Deferred()
     1204
     1205    def dataReceived(self, data):
     1206        pass
     1207
     1208    def connectionLost(self, reason):
     1209        if reason.type == ResponseDone:
     1210            self.finishedDeferred.callback(None)
     1211        else:
     1212            self.finishedDeferred.errback(reason)
     1213
     1214
    11621215class HTTP11ClientProtocol(Protocol):
    11631216    """
    11641217    L{HTTP11ClientProtocol} is an implementation of the HTTP 1.1 client
     
    12431296            return fail(RequestNotSent())
    12441297
    12451298        self._state = 'TRANSMITTING'
    1246         _requestDeferred = maybeDeferred(request.writeTo, self.transport)
    1247         self._finishedRequest = Deferred()
    12481299
    12491300        # Keep track of the Request object in case we need to call stopWriting
    12501301        # on it.
    12511302        self._currentRequest = request
    12521303
    1253         self._transportProxy = TransportProxyProducer(self.transport)
    1254         self._parser = HTTPClientParser(request, self._finishResponse)
    1255         self._parser.makeConnection(self._transportProxy)
    1256         self._responseDeferred = self._parser._responseDeferred
     1304        self._finishedRequest = Deferred()
    12571305
     1306        if request.headers.hasHeader('expect'):
     1307            expectations = request.headers.getRawHeaders('expect')
     1308            _expects100Continue = '100-continue' in [x.lower() for x in
     1309                    expectations]
     1310        else:
     1311            _expects100Continue = False
     1312
     1313        if _expects100Continue:
     1314            # This is synchronous, so jump right into WAITING
     1315            request._writeHeadersTo(self.transport)
     1316
     1317            self._state = 'WAITING'
     1318
     1319            self._setupParser(request)
     1320
     1321            _100ContinueResponseDeferred = self._parser._responseDeferred
     1322
     1323            self._responseDeferred = Deferred()
     1324
     1325            _requestDeferred = Deferred()
     1326
     1327            def cb100ContinueBody(ignored):
     1328                # Start sending the request body
     1329
     1330                self._state = 'TRANSMITTING'
     1331
     1332                self._disconnectParser(None)
     1333
     1334                self._setupParser(request)
     1335
     1336                _requestBodyDeferred = maybeDeferred(
     1337                        request._writeBodyTo, self.transport)
     1338
     1339                _requestBodyDeferred.chainDeferred(_requestDeferred)
     1340
     1341                self._parser._responseDeferred.chainDeferred(
     1342                        self._responseDeferred)
     1343
     1344            def brokenServerTimer():
     1345                # In case the parser hasn't received any data yet, send
     1346                # ourselves a fake 100-Continue response.
     1347
     1348                if not self._parser._hasResponse():
     1349                    self.dataReceived(
     1350                            "HTTP/1.1 100 Continue\r\n"
     1351                            "Content-length: 0\r\n"
     1352                            "\r\n")
     1353
     1354            _brokenServerDelayed = reactor.callLater(TIMEOUT_100_CONTINUE,
     1355                    brokenServerTimer)
     1356
     1357            def ebFirstResponse(err):
     1358                if _brokenServerDelayed.active():
     1359                    _brokenServerDelayed.cancel()
     1360
     1361                # Tell the producer we don't need its body and forward the
     1362                # error
     1363                request.stopWriting()
     1364                self._finishedRequest.errback(err)
     1365
     1366            def cbFirstResponse(response):
     1367                if _brokenServerDelayed.active():
     1368                    _brokenServerDelayed.cancel()
     1369
     1370                if response.code == 100:
     1371                    # Read and discard the response body
     1372
     1373                    discarder = DiscardWithDeferred()
     1374
     1375                    response.deliverBody(discarder)
     1376
     1377                    discarder.finishedDeferred.addCallbacks(cb100ContinueBody,
     1378                            ebFirstResponse)
     1379
     1380                else:
     1381                    # Tell the producer we don't need its body and forward the
     1382                    # response
     1383                    request.stopWriting()
     1384                    self._finishedRequest.callback(response)
     1385
     1386            _100ContinueResponseDeferred.addCallbacks(
     1387                    cbFirstResponse, ebFirstResponse)
     1388
     1389        else:
     1390            _requestDeferred = maybeDeferred(request.writeTo, self.transport)
     1391
     1392            self._setupParser(request)
     1393
     1394            self._responseDeferred = self._parser._responseDeferred
     1395
    12581396        def cbRequestWrotten(ignored):
    12591397            if self._state == 'TRANSMITTING':
    12601398                self._state = 'WAITING'
     
    12791417        return self._finishedRequest
    12801418
    12811419
     1420    def _setupParser(self, request):
     1421        """
     1422        Setup a L{HTTPClientParser} for a L{Response} to a L{Request}. If this
     1423        is not the first parser associated with this protocol, call
     1424        L{HTTP11ClientProtocol._disconnectParser} first.
     1425
     1426        @param request: L{Request} waiting for a L{Response}
     1427        """
     1428        self._transportProxy = TransportProxyProducer(self.transport)
     1429        self._parser = HTTPClientParser(request, self._finishResponse)
     1430        self._parser.makeConnection(self._transportProxy)
     1431
     1432
    12821433    def _finishResponse(self, rest):
    12831434        """
    12841435        Called by an L{HTTPClientParser} to indicate that it has parsed a