[Twisted-Python] DeferredDict, defaultdict, map/reduce, silliness

[Terry Jones]

This isn't meant to be taken too seriously - perhaps just food for someone
else to chew on while trying to relax...

I was thinking the other night about what a DeferredDict might look like,
what it might be useful for, if anything, etc. We have DeferredList of
course, so why not a dict?  Then I also remembered collections.defaultdict
and couldn't resist the appeal of trying defaultdict(defer.Deferred).  That
thinking led to the map/reduce silliness below. I don't know that it leads
anywhere - I was just playing around.

Terry


import sys
from collections import defaultdict
from twisted.internet import defer, reactor

class MapReduce(object):
    def __init__(self, init, cb):
        self.init = init
        self.cb = cb
        self.d = defaultdict(defer.Deferred)

    def map(self, *keys):
        for key in keys:
            self.d[key].addCallback(self.cb)

    def reduce(self):
        def finalize(result, key):
            return (key, result)
        deferreds = []
        for key, d in self.d.iteritems():
            d.addCallback(finalize, key)
            deferreds.append(d)
            d.callback(self.init)
        return defer.gatherResults(deferreds)

def main():
    def printResult(result):
        print result
        reactor.stop()
        
    mr = MapReduce(0, lambda x: x + 1)
    for line in sys.stdin.readlines():
        mr.map(*line.split())
    d = mr.reduce().addCallback(printResult)
    
if __name__ == '__main__':
    reactor.callLater(0, main)
    reactor.run()