Ticket #5732: multicall2.2.patch

File multicall2.2.patch, 18.7 KB (added by braudel, 4 years ago)

corrected as per Glyph's feedback

  • twisted/web/xmlrpc.py

     
    319319                                        ['string', 'string']]
    320320
    321321
     322    @withRequest
     323    def xmlrpc_multicall(self, request, procedureList):
     324        """
     325        Execute several RPC methods in a single XMLRPC request using the
     326        multicall object.
     327
     328        Example:
     329            On the server side, just load the instrospection so your
     330            server has system.multicall. Then on the client:
     331
     332            from twisted.web.xmlrpc import Proxy
     333            from twisted.web.xmlrpc import MultiCall
     334
     335            proxy = Proxy('url of your server')
     336
     337            multiRPC = Multicall( proxy )
     338            # queue a few calls
     339            multiRPC.system.listMethods()
     340            multiRPC.system.methodHelp('system.listMethods')
     341            multiRPC.system.methodSignature('system.listMethods')
     342
     343            def handleResults(results):
     344                for success, result in results:
     345                    print result
     346
     347            multiRPC().addCallback(handleResults)
     348
     349        @param request: the http C{request} object, obtained
     350            via the @withRequest decorator
     351        @type request: L{http.Request}
     352
     353        @param procedureList: A list of dictionaries, each representing an
     354            individual rpc call, containing the C{methodName} and the
     355            C{params}
     356        @type procedureList: list
     357
     358        @return: L{defer.DeferredList} of the deferreds for each procedure
     359            in procedure list
     360        @rtype: L{defer.DeferredList}
     361        """
     362        def callError(error):
     363            """
     364            errorback to handle individual call errors
     365
     366            Individual errors in a multicall are returned as
     367            dictionaries. See U{http://www.xmlrpc.com/discuss/msgReader$1208}.
     368
     369            @param result: C{failure}
     370            @type result: L{Failure}
     371
     372            @rtype: dict
     373            @return: a dict with keys C{faultCode} and C{faultString}
     374            """
     375            log.err(error.value)
     376            return {'faultCode':   self.FAILURE,
     377                    'faultString': (error.value.faultString
     378                        if isinstance(error.value, Fault)
     379                        else getattr(error.value, 'message', ''))}
     380
     381        def prepareCallResponse(result):
     382            """
     383            callback to convert a call C{response} to a list
     384
     385            The xmlrpc multicall spec expects a list wrapping
     386            each call response.
     387            See U{http://www.xmlrpc.com/discuss/msgReader$1208}.
     388
     389            @param result: C{response}
     390            @type result: any python type
     391
     392            @rtype: list
     393            @return: a list with response as element 0
     394            """
     395            return [result]
     396
     397        def run(procedurePath, params):
     398            """
     399            run an individual procedure from the L{procedureList} and
     400            returns a C{deferred}
     401
     402            @param procedurePath: string naming a procedure
     403            @type procedurePath: str
     404
     405            @param params: list of arguments to be passed to the procedure
     406            @type params: list
     407
     408            @return: a C{deferred} object with prepareCallResponse and
     409            callError attached.
     410            @rtype: L{defer.Deferred}
     411            """
     412            try:
     413                procedure = self._xmlrpc_parent.lookupProcedure(procedurePath)
     414            except NoSuchFunction, e:
     415                return defer.fail(e).addErrback(callError)
     416            else:
     417                if getattr(procedure, 'withRequest', False):
     418                    call = defer.maybeDeferred(procedure, request, *params)
     419                else:
     420                    call = defer.maybeDeferred(procedure, *params)
     421
     422                call.addCallback(prepareCallResponse)
     423                call.addErrback(callError)
     424                return call
     425
     426        results = [
     427            run(procedure['methodName'], procedure['params'])
     428            for procedure in procedureList]
     429
     430        return (defer.DeferredList(results)
     431            .addCallback(lambda results: [r[1] for r in results]))
     432
     433    xmlrpc_multicall.signature = [['array', 'array']]
     434
     435
     436
     437class _DeferredMultiCallProcedure(object):
     438    """
     439    A helper object to store calls made on the
     440    MultiCall object for batch execution
     441    """
     442    def __init__(self, call_list, name):
     443        self.__call_list = call_list
     444        self.__name = name
     445
     446
     447    def __getattr__(self, name):
     448        """
     449        magic to emulate x.y.name lookups for
     450        a remote procedure
     451        """
     452        return _DeferredMultiCallProcedure(
     453            self.__call_list,
     454            "%s.%s" % (self.__name, name)
     455        )
     456
     457
     458    def __call__(self, *args):
     459        """
     460        "calling" an RPC on the multicall queues a deferred,
     461        the procedure name, and its calling args.
     462
     463        @return: a L{defer.Deferred} that will be fired when the
     464            results for this RPC are processed
     465        @rtype: L{defer.Deferred}
     466        """
     467        d = defer.Deferred()
     468        self.__call_list.append((d, self.__name, args))
     469        return d
     470
     471
     472
     473class MultiCall(xmlrpclib.MultiCall):
     474    """
     475    @param server: a object used to boxcar method calls
     476    @type server should be a twisted xmlrpc L{Proxy} object.
     477
     478    @return: a L{defer.DeferredList} of all the deferreds for
     479        each queued rpc call
     480    @rtype: L{defer.DeferredList}
     481
     482    Methods can be added to the MultiCall using normal
     483    method call syntax e.g.:
     484
     485    proxy = Proxy('http://advogato.org/XMLRPC')
     486
     487    multicall = MultiCall(proxy)
     488    d1 = multicall.add(2,3)
     489    d2 = multicall.add(5,6)
     490
     491    or
     492
     493    d3 = multicall.callRemote('add', 2, 3)
     494
     495    To execute the multicall, call the MultiCall object
     496    and attach callbacks, errbacks to the returned
     497    deferred e.g.:
     498
     499    def printResults(results):
     500        for result in results:
     501            print result[1]
     502
     503    d = multicall()
     504    d.addCallback(printResults)
     505    """
     506    def __getattr__(self, name):
     507        """ get a ref to a helper object to emulate
     508        RPC 'attributes lookup'
     509        """
     510        return _DeferredMultiCallProcedure(self.__call_list, name)
     511
     512
     513    def callRemote(self, method, *args):
     514        """
     515        queue a call for c{method} on this multicall object
     516        with given arguments.
     517
     518        @return: a L{defer.Deferred} that will fire with the method response,
     519            or a failure if the method failed.
     520        """
     521        return getattr(self, method)(*args)
     522
     523
     524    def __call__(self):
     525        """
     526        execute the multicall, processing the deferreds for each
     527        procedure once the results are ready
     528
     529        @return: a L{defer.DeferredList} that will fire all the queued deferreds
     530        """
     531        marshalled_list = []
     532        deferreds = []
     533        for deferred, name, args in self.__call_list:
     534            marshalled_list.append({
     535                'methodName': name,
     536                'params': args})
     537            deferreds.append(deferred)
     538
     539        def processResults(results, deferreds):
     540            """
     541            callback to return an xmlrpclib
     542            MultiCallIterator of the results
     543            """
     544            for d, result in zip(deferreds, results):
     545                if isinstance(result, dict):
     546                    d.errback(Fault(result['faultCode'],
     547                        result['faultString']))
     548
     549                elif isinstance(result, list):
     550                    d.callback(result[0])
     551
     552                else:
     553                    raise ValueError(
     554                        "unexpected type in multicall result")
     555
     556        self.__server.callRemote(
     557            'system.multicall', marshalled_list
     558        ).addCallback(processResults, deferreds)
     559
     560        return defer.DeferredList(deferreds)
     561
     562
    322563def addIntrospection(xmlrpc):
    323564    """
    324565    Add Introspection support to an XMLRPC server.
  • twisted/web/test/test_xmlrpc.py

     
    1414from twisted.web import xmlrpc
    1515from twisted.web.xmlrpc import (
    1616    XMLRPC, payloadTemplate, addIntrospection, _QueryFactory, Proxy,
    17     withRequest)
     17    withRequest, MultiCall)
    1818from twisted.web import server, static, client, error, http
    1919from twisted.internet import reactor, defer
    2020from twisted.internet.error import ConnectionDone
     
    2828else:
    2929    sslSkip = None
    3030
    31 
    3231class AsyncXMLRPCTests(unittest.TestCase):
    3332    """
    3433    Tests for L{XMLRPC}'s support of Deferreds.
     
    667666        return d
    668667
    669668
     669
    670670class XMLRPCTestIntrospection(XMLRPCTestCase):
    671671
    672672    def setUp(self):
     
    686686                 'deferFault', 'dict', 'echo', 'fail', 'fault',
    687687                 'pair', 'system.listMethods',
    688688                 'system.methodHelp',
    689                  'system.methodSignature', 'withRequest'])
     689                 'system.methodSignature', 'system.multicall',
     690                 'withRequest'])
    690691
    691692        d = self.proxy().callRemote("system.listMethods")
    692693        d.addCallback(cbMethods)
     
    720721        return defer.DeferredList(dl, fireOnOneErrback=True)
    721722
    722723
     724
     725class FakeProxy(object):
     726    """
     727    Fake twisted XMLRPC Proxy client to run tests without using
     728    the network
     729    """
     730    def __init__(self, resource):
     731        self.resource = resource
     732
     733
     734    def callRemote(self, methodName, *args):
     735        """
     736        emulate twisted.web.xmlrpc.Proxy.callRemote
     737        """
     738        # build request
     739        request = DummyRequest([''])
     740        request.method = 'POST'
     741        request.content = StringIO(
     742            payloadTemplate % (methodName, xmlrpclib.dumps(args)))
     743       
     744        def returnResponse( requestResponse ):
     745            results = xmlrpclib.loads(requestResponse)[0]
     746            if len(results) == 1:
     747                results = results[0]
     748            return results
     749
     750        # look mom no network!
     751        self.resource.render(request)
     752
     753        return (defer.succeed("".join(request.written))
     754            .addCallback(returnResponse))
     755
     756
     757
     758class XMLRPCTestMultiCall(unittest.TestCase):
     759    """
     760    Tests for xmlrpc multicalls
     761    """
     762    def setUp(self):
     763        self.resource = Test()
     764        addIntrospection(self.resource)
     765        self.proxy = FakeProxy(self.resource)
     766
     767
     768    def test_multicall(self):
     769        """
     770        test a suscessfull multicall
     771        """
     772        inputs = range(5)
     773        m = MultiCall(self.proxy)
     774        for x in inputs:
     775            m.echo(x)
     776
     777        def testResults(results):
     778            self.assertEqual(inputs, [x[1] for x in results])
     779
     780        resultsDeferred = m().addCallback(testResults)
     781        self.assertTrue(resultsDeferred.called)
     782
     783
     784    def test_multicall_callRemote(self):
     785        """
     786        test a suscessfull multicall using
     787        multicall.callRemote instead of attribute lookups
     788        """
     789        inputs = range(5)
     790        m = MultiCall(self.proxy)
     791        for x in inputs:
     792            m.callRemote('echo', x)
     793
     794        def testResults(results):
     795            self.assertEqual(inputs, [x[1] for x in results])
     796
     797        resultsDeferred = m().addCallback(testResults)
     798        self.assertTrue(resultsDeferred.called)
     799
     800
     801    def test_multicall_with_callbacks(self):
     802        """
     803        test correct execution of callbacks added to the
     804        multicall's returned deferreds for each individual queued
     805        call
     806        """
     807        inputs = range(5)
     808        m = MultiCall(self.proxy)
     809        for x in inputs:
     810            d = m.echo(x)
     811            d.addCallback( lambda x : x*x )
     812
     813        def testResults(results):
     814            self.assertEqual([ x*x for x in inputs], [x[1] for x in results])
     815
     816        resultsDeferred = m().addCallback(testResults)
     817        self.assertTrue(resultsDeferred.called)
     818
     819
     820    def test_multicall_errorback(self):
     821        """
     822        test that an error (an invalid - not found - method)
     823        does not propagate if properly handled in the errorback
     824        of an individual deferred
     825        """
     826        def trapFoo(error):
     827            error.trap(xmlrpclib.Fault)
     828            self.assertEqual(error.value.faultString,
     829                'procedure foo not found',
     830                'check we have a failure message'
     831                )
     832            self.flushLoggedErrors(xmlrpc.NoSuchFunction)
     833
     834
     835        m = MultiCall(self.proxy)
     836        m.echo(1)
     837        # method not present on server
     838        m.foo().addErrback(trapFoo)
     839        m.echo(2)
     840
     841        def handleErrors(error):
     842            error.trap(xmlrpclib.Fault)
     843            self.assertEqual(error.value.faultString,
     844                'xmlrpc_echo() takes exactly 2 arguments (4 given)')
     845            self.flushLoggedErrors(TypeError)
     846
     847        m.echo(1,2,3).addErrback(handleErrors)
     848
     849        def testResults(results):
     850            """
     851            the errorback should have trapped the error
     852            """
     853            self.assertEqual(results[1], (True, None),
     854            'failure trapped in errorback does not propagate to deferredList results')
     855
     856        resultsDeferred = m().addCallback(testResults)
     857        self.assertTrue(resultsDeferred.called)
     858
     859
     860    def test_multicall_withRequest(self):
     861        """
     862        Test that methods decorated with @withRequest are handled correctly
     863        """
     864        m = MultiCall(self.proxy)
     865        m.echo(1)
     866        # method decorated with withRequest
     867        msg = 'hoho'
     868        m.withRequest(msg)
     869        m.echo(2)
     870
     871        def testResults(results):
     872            """
     873            test that a withRequest decorated method was properly handled
     874            """
     875            self.assertEqual(results[1][1],
     876                'POST %s' % msg, 'check withRequest decorated result')
     877
     878        resultsDeferred = m().addCallback(testResults)
     879        self.assertTrue(resultsDeferred.called)
     880
     881
     882    def test_multicall_with_xmlrpclib(self):
     883        """
     884        check that the sever's response is also compatible with xmlrpclib
     885        MultiCall client
     886        """
     887        class PatchedXmlrpclibProxy(object):
     888            """
     889            A proxy that more closely resembles xmlrpclib.ServerProxy
     890            """
     891            def __init__(self, resource):
     892                self.resource = resource
     893
     894            def __request(self, methodName, params):
     895                """
     896                Patched xmlrpclib.ServerProxy.__request to emulate
     897                RPC call without using the network
     898                """
     899                request = DummyRequest([''])
     900                request.method = 'POST'
     901                request.content = StringIO(
     902                    payloadTemplate % (methodName, xmlrpclib.dumps(params)))
     903
     904                self.resource.render(request)
     905                response =  xmlrpclib.loads("".join(request.written))[0]
     906                if len(response) == 1:
     907                    response = response[0]
     908                return response
     909
     910            def __getattr__(self, name):
     911                """
     912                magic method dispatcher
     913                """
     914                return xmlrpclib._Method(self.__request, name)
     915
     916        inputs = range(5)
     917        m = xmlrpclib.MultiCall(
     918            PatchedXmlrpclibProxy(self.resource))
     919        for x in inputs:
     920            m.echo(x)
     921
     922        self.assertEqual(
     923                inputs,
     924                list(m()),
     925                'xmlrpclib multicall can talk to the twisted multicall')
     926
     927
     928
    723929class XMLRPCClientErrorHandling(unittest.TestCase):
    724930    """
    725931    Test error handling on the xmlrpc client.
  • doc/web/howto/xmlrpc.xhtml

     
    234234no <code class="python">help</code> attribute is specified, the
    235235method's documentation string is used instead.</p>
    236236
     237
     238 
    237239<h2>SOAP Support</h2>
    238240
    239241<p>From the point of view of a Twisted developer, there is little difference
     
    295297[8, 15]
    296298</pre>
    297299
     300<h3>Using multicall Objects</h3>
     301<p>Another informal pattern commonly used along with xmlrpc
     302introspection is the use of a multicall object to boxcar several
     303rpc calls in a single http request.
     304
     305An instance of a <code>MultiCall</code> object
     306is created on the client side and several RPC calls are queued by
     307calling them on this <code>MultiCall</code> instance. Every RPC called on the MultiCall returns a <code>deferred</code> that you can use to attach callbacks to each queued rpc response.
     308
     309Executing the multicall instance itself triggers a request to
     310<code>system.multicall</code> with a list of dictionaries representing each of the procedures to call, returning a <code>DeferredList</code> of the deferreds corresponding to each individual call.
     311
     312Let's see an example to talk to the server above using a MultiCall:</p>
     313
     314<pre class="python">
     315from twisted.web.xmlrpc import Proxy, MultiCall, NoSuchFunction
     316from twisted.internet import reactor
     317
     318def printResults(results):
     319    for result in results:
     320        print result[1]
     321    reactor.stop()
     322
     323proxy = Proxy('http://127.0.0.1:7080')
     324
     325multiRPC = MultiCall(proxy)
     326
     327# queue a few echo calls, with a callback to square the results
     328for x in range(10):
     329    multiRPC.echo(x).addCallback(lamdba x: x*x)
     330
     331# you can of course use the <code>callRemote</code> method instead
     332# for a better twisted-like feeling
     333for x in range(10,20):
     334    multiRPC.callRemote('echo', x).addCallback(lamdba x: x*x)
     335
     336# now queue a few more echo calls with a callback that will
     337# indicate the echoed letter and their index in the original string
     338for index, x in enumerate('hello'):
     339    multiRPC.echo(x).addCallback(lambda result, index: '%s found at %d' % (result, index))
     340
     341# individual deferreds ideally should also handle errors
     342def handleErrors(error):
     343    error.trap(xmlrpclib.Fault)
     344    print "yay! The server said %s" % error.value.faultString
     345multiRPC.foo().addErrback(handleErrors)
     346
     347# finally execute the multiRPC instance, to send the request,
     348# then handle the returned DeferredList
     349# results with the printResults callback
     350multiRPC().addCallback(printResults)
     351reactor.run()
     352</pre>
     353
     354The <code>system.multicall</code> method is also 100% compatible with the xmlrpclib MultiCall, so you can also use xmlrpclib as the client:
     355
     356<pre class="python">
     357from xmlrpclib import ServerProxy, Multicall
     358
     359proxy = ServerProxy('http://127.0.0.1:7080')
     360
     361multiRPC = MultiCall(proxy)
     362for x in range(10):
     363    multiRPC.echo(x)
     364
     365for result in multiRPC():
     366    print result * result
     367</pre>
     368
     369
    298370<h2>Serving SOAP and XML-RPC simultaneously</h2>
    299371
    300372<p><code class="API">twisted.web.xmlrpc.XMLRPC</code> and <code