[Twisted-Python] Transparent pooling of deferreds to be fired upon another deferred firing

Terry Jones terry at jon.es
Thu Apr 23 13:00:35 EDT 2009


BTW, below is a version of my code from yesterday that works on class
methods.  I didn't write the __get__ code, but took it from a decorator
tutorial on the web.

This version works with standalone functions, and with class methods that
also use @inlineCallbacks.  So you can do

    class MyClass(object):
        @DeferredPooler
        @defer.inlineCallbacks
        def xxx(self, ...):
            d = funcReturningADeferred()
            d.addCallbacks(...)
            yield d

and it works.  I also wrote a small test suite in case anyone wants it.
There are still some issues (e.g., with timeouts), but I've decorated some
methods with DeferredPooler and am happy with the result so far.

Terry


    from twisted.internet import defer
    from twisted.python import failure

    class DeferredPooler(object):
        def __init__(self, func):
            self.pool = {}
            self.func = func

        def _callOthers(self, result, key):
            if isinstance(result, failure.Failure):
                for d in self.pool[key]:
                    d.errback(result)
            else:
                for d in self.pool[key]:
                    d.callback(result)
            del self.pool[key]
            return result

        def __call__(self, *args, **kwargs):
            key = (args, hash(tuple(sorted(kwargs.items()))))
            if key in self.pool:
                d = defer.Deferred()
                self.pool[key].append(d)
                return d
            else:
                self.pool[key] = []
                d = self.func(*args, **kwargs)
                assert isinstance(d, defer.Deferred)
                d.addBoth(self._callOthers, key)
                return d

        def __get__(self, obj, type=None):
            if obj is None:
                return self
            new_func = self.func.__get__(obj, type)
            return self.__class__(new_func)




More information about the Twisted-Python mailing list