Ticket #2245: cdefer.3.patch

twisted/test/test_defer.py

@@ -45 +45 @@
     def _errback(self, *args, **kw):
         self.errback_results = args, kw
 
+    def testNoResult(self):
+        deferred = defer.Deferred()
+        self.assertRaises(AttributeError, lambda : deferred.result)
+
+    def testReentrantRunCallbacks(self):
+        deferred = defer.Deferred()
+        called = []
+        def callback2(result):
+            called.append(2)
+        def callback1(result):
+            called.append(1)
+            deferred.addCallback(callback2)
+        deferred.addCallback(callback1)
+        deferred.callback(None)
+        self.assertEquals(called, [1, 2])
+
+    def testAddCallbackErrbackWithoutArgs(self):
+        deferred = defer.Deferred()
+        self.assertRaises(TypeError, deferred.addCallback)
+        self.assertRaises(TypeError, deferred.addErrback)
+        self.assertRaises(TypeError, deferred.addBoth)
+        # assertRaises gives a **{} to the function, whereas no arg at
+        # all gives a NULL, so lets implement it manually here:
+        for cback in [defer.Deferred.addCallback,
+                      defer.Deferred.addErrback,
+                      defer.Deferred.addBoth]:
+            try:
+                cback(deferred)
+            except TypeError, e:
+                pass
+            else:
+                self.assert_(False, "addCallback did not raise TypeError")
+
     def testCallbackWithoutArgs(self):
         deferred = defer.Deferred()
         deferred.addCallback(self._callback)
@@ -562 +595 @@
         def _subErrorLogWithInnerFrameCycle():
             d = defer.Deferred()
             d.addCallback(lambda x, d=d: 1/0)
-            d._d = d
             d.callback(1)
+            d.result = d
 
         _subErrorLogWithInnerFrameCycle()
         gc.collect()

twisted/mail/alias.py

@@ -210 +210 @@
     def eomReceived(self):
         if not self.done:
             self.protocol.transport.loseConnection()
-            self.completion.setTimeout(60)
+            from twisted.internet import reactor
+            def timeout_cb():
+                self.completion.errback(defer.TimeoutError())
+            timeout = reactor.callLater(60, timeout_cb)
+            def cancel_timeout(result):
+                timeout.cancel()
+                return result
+            self.completion.addBoth(cancel_timeout)
         return self.completion
     
     def connectionLost(self):

twisted/topfiles/setup.py

@@ -34 +34 @@
     # Extension modules to build.
     exts = []
 
+    # twisted.internet.cdefer implementation of Deferred
+    exts.append( Extension('twisted.internet.cdefer',
+                           ['twisted/internet/cdefer/cdefer.c']) )
+
+
     # urllib.unquote accelerator
     exts.append( Extension("twisted.protocols._c_urlarg",
                             ["twisted/protocols/_c_urlarg.c"],

twisted/internet/cdefer/cdefer.c

@@ +1 @@
+/*
+ * A Deferred implementation in C. Cover most of the Deferred API but try
+ * to be faster.
+ *
+ * TODO: Review failure handling - is everything decref'd properly?
+ * TODO: Try to replace IsInstance calls with a fast ptr comparison?
+ *
+ */
+
+#include <Python.h>
+#include "structmember.h"
+
+/* Py_VISIT and Py_CLEAR are defined here to be compatible with Python 2.3 */
+
+#ifndef Py_VISIT
+#define Py_VISIT(op) \
+    do { \
+        if (op) { \
+            int vret = visit((PyObject *)(op), arg); \
+            if (vret) \
+                return vret; \
+        } \
+    } while (0)
+#endif
+
+#ifndef Py_CLEAR
+#define Py_CLEAR(op) \
+    do { \
+        if (op) { \
+            PyObject *tmp = (PyObject *)(op); \
+            (op) = NULL; \
+            Py_DECREF(tmp); \
+        } \
+    } while (0)
+#endif
+
+PyObject * failure_class = NULL;
+PyObject * already_called = NULL;
+PyObject * debuginfo_class = NULL;
+
+typedef struct {
+    PyObject_HEAD
+    PyObject *result;
+    int paused;
+    PyObject *callbacks;
+    PyObject *debuginfo;
+    int called;
+    /* Current callback index in the callbacks list to run. This
+     * allows clearing the list once per runCallbacks rather than
+     * popping every item. It has to be per-deferred, because
+     * runCallbacks can be called reentrantly, and must not repeat the
+     * previously-called callbacks. */
+    Py_ssize_t callback_index;
+} cdefer_Deferred;
+
+/* Prototypes */
+
+static PyObject * cdefer_Deferred_new(PyTypeObject *type, PyObject *args,
+        PyObject *kwargs);
+
+static void cdefer_Deferred_dealloc(PyObject *o);
+
+static int cdefer_Deferred_traverse(PyObject *o, visitproc visit, void *arg);
+
+static int cdefer_Deferred_clear(PyObject *o);
+
+static int cdefer_Deferred_clear(PyObject *o);
+
+static int cdefer_Deferred___init__(cdefer_Deferred *self, PyObject *args,
+        PyObject *kwargs);
+
+static PyObject *cdefer_Deferred__addCallbacks(cdefer_Deferred *self,
+        PyObject *callback, PyObject *errback, PyObject *callbackArgs,
+        PyObject *callbackKeywords, PyObject *errbackArgs,
+        PyObject *errbackKeywords);
+
+static PyObject *cdefer_Deferred_addCallback(cdefer_Deferred *self,
+        PyObject *args, PyObject *kwargs);
+
+static PyObject *cdefer_Deferred_addErrback(cdefer_Deferred *self,
+        PyObject *args, PyObject *kwargs);
+
+static PyObject *cdefer_Deferred_addBoth(cdefer_Deferred *self, PyObject *args,
+        PyObject *kwargs);
+
+static PyObject *cdefer_Deferred_pause(cdefer_Deferred *self, PyObject *args);
+
+static PyObject *cdefer_Deferred_unpause(cdefer_Deferred *self,
+        PyObject *args);
+
+static PyObject *cdefer_Deferred_chainDeferred(cdefer_Deferred *self,
+        PyObject *args, PyObject *kwargs);
+
+static PyObject *cdefer_Deferred__runCallbacks(cdefer_Deferred *self);
+
+static PyObject *cdefer_Deferred__startRunCallbacks(cdefer_Deferred *self,
+        PyObject *result);
+
+static PyObject *cdefer_Deferred_callback(cdefer_Deferred *self, PyObject *args,
+        PyObject *kwargs);
+
+static PyObject *cdefer_Deferred_errback(cdefer_Deferred *self, PyObject *args,
+        PyObject *kwargs);
+
+static PyObject *cdefer_Deferred__continue(cdefer_Deferred *self,
+        PyObject *args, PyObject *kwargs);
+
+static PyTypeObject cdefer_DeferredType;
+
+static PyObject * cdefer_Deferred_new(PyTypeObject *type, PyObject *args,
+        PyObject *kwargs) {
+    cdefer_Deferred *self;
+    self = (cdefer_Deferred *)type->tp_alloc(type, 0);
+    return (PyObject *)self;
+}
+
+static void cdefer_Deferred_dealloc(PyObject *o) {
+    cdefer_Deferred *self;
+    self = (cdefer_Deferred *)o;
+    PyObject_GC_UnTrack(self);
+    Py_XDECREF(self->result);
+    Py_XDECREF(self->debuginfo);
+    Py_XDECREF(self->callbacks);
+    (*o->ob_type->tp_free)(o);
+}
+
+static int cdefer_Deferred_traverse(PyObject *o, visitproc visit, void *arg) {
+    cdefer_Deferred *self;
+    self = (cdefer_Deferred *)o;
+    Py_VISIT(self->result);
+    Py_VISIT(self->debuginfo);
+    Py_VISIT(self->callbacks);
+    return 0;
+}
+
+static int cdefer_Deferred_clear(PyObject *o) {
+    cdefer_Deferred *self;
+    self = (cdefer_Deferred *)o;
+    Py_CLEAR(self->result);
+    Py_CLEAR(self->debuginfo);
+    Py_CLEAR(self->callbacks);
+    return 0;
+}
+
+static int cdefer_Deferred___init__(cdefer_Deferred *self, PyObject *args,
+        PyObject *kwargs) {
+    static char *argnames[] = {NULL};
+    if (!PyArg_ParseTupleAndKeywords(args, kwargs, "", argnames)) {
+        return -1;
+    }
+    self->paused = 0;
+    self->callback_index = 0;
+    self->callbacks = PyList_New(0);
+    if (!self->callbacks) {
+        return -1;
+    }
+    return 0;
+}
+
+static PyObject *cdefer_Deferred__addCallbacks(cdefer_Deferred *self,
+        PyObject *callback, PyObject *errback, PyObject *callbackArgs,
+        PyObject *callbackKeywords, PyObject *errbackArgs,
+        PyObject *errbackKeywords) {
+    PyObject *result;
+    PyObject *cbs = 0;
+    int rc;
+
+    if (callback != Py_None) {
+        if (!PyCallable_Check(callback)) {
+            PyErr_SetNone(PyExc_AssertionError);
+            return NULL;
+        }
+    }
+    if (errback != Py_None) {
+        if (!PyCallable_Check(errback)) {
+            PyErr_SetNone(PyExc_AssertionError);
+            return NULL;
+        }
+    }
+
+    cbs = Py_BuildValue("(OOOOOO)", callback, callbackArgs, callbackKeywords,
+                                    errback, errbackArgs, errbackKeywords);
+    if (!cbs) {
+        return NULL;
+    }
+
+    rc = PyList_Append(self->callbacks, cbs);
+    Py_CLEAR(cbs);
+    if (rc == -1) {
+        return NULL;
+    }
+
+    if (self->called) {
+        if (cdefer_Deferred__runCallbacks(self) == NULL) {
+            return NULL;
+        }
+    }
+
+    result = (PyObject *)self;
+    Py_INCREF(result);
+    return result;
+}
+
+static char cdefer_Deferred_addCallbacks_doc[] = "Add a pair of callbacks (success and error) to this Deferred.\n\nThese will be executed when the \'master\' callback is run.";
+
+static PyObject *cdefer_Deferred_addCallbacks(cdefer_Deferred *self,
+        PyObject *args, PyObject *kwargs) {
+    static char *argnames[] = {"callback", "errback", "callbackArgs",
+        "callbackKeywords", "errbackArgs", "errbackKeywords", NULL};
+    PyObject *callback;
+    PyObject *errback = Py_None;
+    PyObject *callbackArgs = Py_None;
+    PyObject *callbackKeywords = Py_None;
+    PyObject *errbackArgs = Py_None;
+    PyObject *errbackKeywords = Py_None;
+    PyObject *result;
+    if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|OOOOO", argnames,
+                &callback, &errback, &callbackArgs,
+                &callbackKeywords, &errbackArgs, &errbackKeywords)) {
+        return NULL;
+    }
+    result = cdefer_Deferred__addCallbacks(self, callback, errback,
+        callbackArgs, callbackKeywords, errbackArgs, errbackKeywords);
+    return result;
+}
+
+static char cdefer_Deferred_addCallback_doc[] = "Convenience method for adding just a callback.\n\nSee L{addCallbacks}.";
+
+/* Returns a NEW reference to the callback/errback arg, and to the
+ * cbackArgs, but a BORROWED reference to the keywords. In case of
+ * error, no references are returned/touched */
+static PyObject *extract_cback_args_kw(char *argname,
+                                       PyObject *args, PyObject *kwargs,
+                                       PyObject **cbackArgs,
+                                       PyObject **cbackKeywords)
+{
+    PyObject *cback;
+    
+    if (kwargs) {
+        (*cbackKeywords) = kwargs;
+    } else {
+        (*cbackKeywords) = Py_None;
+    }
+    if (PyTuple_Size(args) > 0) {
+        cback = PyTuple_GET_ITEM(args, 0);
+        if (!cback) {
+            return NULL;
+        }
+        (*cbackArgs) = PyTuple_GetSlice(args, 1, PyTuple_Size(args));
+        if (!(*cbackArgs)) {
+            return NULL;
+        }
+        Py_INCREF(cback);
+    } else {
+        cback = PyDict_GetItemString((*cbackKeywords), argname);
+        if (!cback) {
+            PyErr_Format(PyExc_TypeError, "addCallback requires '%s' argument'", argname);
+            return NULL;
+        }
+        (*cbackArgs) = Py_None;
+        Py_INCREF(Py_None);
+
+        /* "callback" in the keyword dict may be the only reference to
+         * it, and we delete it from the dict, so we must own a
+         * reference too */
+        Py_INCREF(cback);
+
+        if (PyDict_DelItemString((*cbackKeywords), argname) == -1) {
+            Py_DECREF(cback);
+            Py_DECREF(Py_None);
+            return NULL;
+        }
+    }
+    return cback;
+}
+
+static PyObject *cdefer_Deferred_addCallback(cdefer_Deferred *self,
+        PyObject *args, PyObject *kwargs) {
+    PyObject *callback;
+    PyObject *callbackArgs;
+    PyObject *callbackKeywords;
+    PyObject *result;
+    callback = extract_cback_args_kw("callback", args, kwargs, &callbackArgs, &callbackKeywords);
+    if (!callback) {
+        return NULL;
+    }
+    result = cdefer_Deferred__addCallbacks(self, callback, Py_None, callbackArgs,
+        callbackKeywords, Py_None, Py_None);
+    Py_DECREF(callback);
+    Py_DECREF(callbackArgs);
+    return result;
+}
+
+static char cdefer_Deferred_addErrback_doc[] = "Convenience method for adding just an errback.\n\nSee L{addCallbacks}.";
+
+static PyObject *cdefer_Deferred_addErrback(cdefer_Deferred *self,
+        PyObject *args, PyObject *kwargs) {
+    PyObject *errback;
+    PyObject *errbackArgs;
+    PyObject *errbackKeywords;
+    PyObject *result;
+    errback = extract_cback_args_kw("errback", args, kwargs, &errbackArgs, &errbackKeywords);
+    if (!errback) {
+        return NULL;
+    }
+    result = cdefer_Deferred__addCallbacks(self, Py_None, errback, Py_None,
+        Py_None, errbackArgs, errbackKeywords);
+    Py_DECREF(errback);
+    Py_DECREF(errbackArgs);
+    return result;
+}
+
+static char cdefer_Deferred_addBoth_doc[] = "Convenience method for adding a single callable as both a callback\nand an errback.\n\nSee L{addCallbacks}.";
+
+static PyObject *cdefer_Deferred_addBoth(cdefer_Deferred *self, PyObject *args,
+        PyObject *kwargs) {
+    PyObject *callback;
+    PyObject *callbackArgs;
+    PyObject *callbackKeywords;
+    PyObject *result;
+    callback = extract_cback_args_kw("callback", args, kwargs, &callbackArgs, &callbackKeywords);
+    if (!callback) {
+        return NULL;
+    }
+    result = cdefer_Deferred__addCallbacks(self, callback, callback,
+        callbackArgs, callbackKeywords, callbackArgs, callbackKeywords);
+    Py_DECREF(callback);
+    Py_DECREF(callbackArgs);
+    return result;
+}
+
+static char cdefer_Deferred_pause_doc[] = "Stop processing on a Deferred until L{unpause}() is called.";
+
+static PyObject *cdefer_Deferred_pause(cdefer_Deferred *self, PyObject *args) {
+    PyObject *result;
+    self->paused++;
+    result = Py_None;
+    Py_INCREF(Py_None);
+    return result;
+}
+
+static char cdefer_Deferred_unpause_doc[] = "Process all callbacks made since L{pause}() was called.";
+
+static PyObject *cdefer_Deferred_unpause(cdefer_Deferred *self,
+        PyObject *args) {
+    self->paused--;
+    if (!self->paused && self->called) {
+        return cdefer_Deferred__runCallbacks(self);
+    }
+    Py_INCREF(Py_None);
+    return Py_None;
+}
+
+static char cdefer_Deferred_chainDeferred_doc[] = "Chain another Deferred to this Deferred.\n\nThis method adds callbacks to this Deferred to call d\'s callback or\nerrback, as appropriate.";
+
+static PyObject *cdefer_Deferred_chainDeferred(cdefer_Deferred *self,
+        PyObject *args, PyObject *kwargs) {
+    PyObject *d;
+    PyObject *callback;
+    PyObject *errback;
+    PyObject *result;
+    static char *argnames[] = {"d", NULL};
+    if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O", argnames, &d)) {
+        return NULL;
+    }
+    callback = PyObject_GetAttrString(d, "callback");
+    if (!callback) {
+        return NULL;
+    }
+    errback = PyObject_GetAttrString(d, "errback");
+    if (!errback) {
+        Py_DECREF(callback);
+        return NULL;
+    }
+    result = cdefer_Deferred__addCallbacks(self, callback, errback, Py_None,
+        Py_None, Py_None, Py_None);
+    Py_DECREF(callback);
+    Py_DECREF(errback);
+    return result;
+}
+
+static int cdefer_Deferred__set_debuginfo_fail_result(cdefer_Deferred *self) {
+    if (!self->debuginfo) {
+        self->debuginfo = PyObject_CallObject(debuginfo_class, NULL);
+        if (!self->debuginfo) {
+            return -1;
+        }
+    }
+    if (PyObject_SetAttrString(self->debuginfo, "failResult", self->result) == -1) {
+        return -1;
+    }
+    return 0;
+}
+
+static int cdefer_Deferred__clear_debuginfo(cdefer_Deferred *self) {
+    if (self->debuginfo) {
+        if (PyObject_SetAttrString(self->debuginfo, "failResult", Py_None) == -1) {
+            return -1;
+        }
+    }
+    return 0;
+}
+
+
+static PyObject *cdefer_Deferred__runCallbacks(cdefer_Deferred *self) {
+    PyObject *cb;
+    PyObject *item;
+    PyObject *callback;
+    PyObject *args;
+    PyObject *newArgs;
+    PyObject *newArgs2;
+    PyObject *kwargs;
+    PyObject *_continue;
+    PyObject *type, *value, *traceback, *failArgs;
+    PyObject *tmp;
+    PyObject *result;
+    int size;
+    int offset;
+
+    if (!self->paused) {
+        cb = self->callbacks;
+        for (;;) {
+            size = PyList_GET_SIZE(cb);
+            if (size == -1) {
+                return NULL;
+            }
+            if (self->callback_index >= size) {
+                break;
+            }
+            
+            item = PyList_GET_ITEM(cb, self->callback_index);
+            if (!item) {
+                return NULL;
+            }
+
+            offset = 0;
+            if (PyObject_IsInstance(self->result, failure_class)) {
+                offset = 3;
+            }
+
+            callback = PyTuple_GET_ITEM(item, offset + 0);
+            if (!callback) {
+                return NULL;
+            }
+            if (callback == Py_None) {
+                ++self->callback_index;
+                continue;
+            }
+
+            args = PyTuple_GET_ITEM(item, offset + 1);
+            if (!args) {
+                return NULL;
+            }
+
+            kwargs = PyTuple_GET_ITEM(item, offset + 2);
+            if (!kwargs) {
+                return NULL;
+            }
+
+            newArgs = Py_BuildValue("(O)", self->result);
+            if (!newArgs) {
+                return NULL;
+            }
+
+            if (args != Py_None) {
+                newArgs2 = PySequence_InPlaceConcat(newArgs, args);
+                if (!newArgs2) {
+                    return NULL;
+                }
+                Py_CLEAR(newArgs);
+            } else {
+                newArgs2 = newArgs;
+                newArgs = NULL;
+            }
+
+            ++self->callback_index;
+            if (kwargs == Py_None) {
+                tmp = PyObject_Call(callback, newArgs2, NULL);
+            } else {
+                tmp = PyObject_Call(callback, newArgs2, kwargs);
+            }
+            Py_DECREF(self->result);
+            self->result = tmp;
+
+            Py_CLEAR(newArgs2);
+
+            if (!self->result) {
+                PyErr_Fetch(&type, &value, &traceback);
+                PyErr_NormalizeException(&type, &value, &traceback);
+                if (!traceback) {
+                    traceback = Py_None;
+                    Py_INCREF(traceback);
+                }
+
+                failArgs = Py_BuildValue("(OOO)", value, type, traceback);
+                if (!failArgs) {
+                    PyErr_Restore(type, value, traceback);
+                    return NULL;
+                }
+                self->result = PyObject_CallObject(failure_class, failArgs);
+                Py_DECREF(failArgs);
+                if (!self->result) {
+                    PyErr_Restore(type, value, traceback);
+                    return NULL;
+                }
+                Py_DECREF(type);
+                Py_DECREF(value);
+                Py_DECREF(traceback);
+                continue;
+            }
+            Py_INCREF(self->result);
+            if (PyObject_TypeCheck(self->result, &cdefer_DeferredType)) {
+                if (PyList_SetSlice(cb, 0, self->callback_index, NULL) == -1) {
+                    return NULL;
+                }
+                self->callback_index = 0;
+                
+                result = PyObject_CallMethod((PyObject *)self, "pause", NULL);
+                if (!result) {
+                    return NULL;
+                }
+                Py_DECREF(result);
+                
+                _continue = PyObject_GetAttrString((PyObject *)self,
+                                                   "_continue");
+                if (!_continue) {
+                    return NULL;
+                }
+
+                result = cdefer_Deferred__addCallbacks(
+                    (cdefer_Deferred *)self->result, _continue,
+                    _continue, Py_None, Py_None, Py_None, Py_None);
+                /* The reference was either copied/incref'd or not
+                 * (when errored) in addCallbacks, either way, we own
+                 * one too, and don't need it anymore. */
+                Py_DECREF(_continue);
+                
+                if (!result) {
+                    return NULL;
+                }
+                Py_DECREF(result);
+                
+                goto endLabel;
+            }
+        }
+        if (PyList_SetSlice(cb, 0, PyList_GET_SIZE(cb), NULL) == -1) {
+            return NULL;
+        }
+        self->callback_index = 0;
+    }
+endLabel:;
+    if (PyObject_IsInstance(self->result, failure_class)) {
+        result = PyObject_CallMethod((PyObject *)self->result,
+                                     "cleanFailure", NULL);
+        if (!result) {
+            return NULL;
+        }
+        Py_DECREF(result);
+        if (cdefer_Deferred__set_debuginfo_fail_result(self) == -1) {
+            return NULL;
+        }
+    } else {
+        if (cdefer_Deferred__clear_debuginfo(self) == -1) {
+            return NULL;
+        }
+    }
+    Py_INCREF(Py_None);
+    return Py_None;
+}
+
+static PyObject *cdefer_Deferred__startRunCallbacks(cdefer_Deferred *self,
+        PyObject *result) {
+    if (self->called) {
+        PyErr_SetNone(already_called);
+        return NULL;
+    }
+    self->called = 1;
+    Py_XDECREF(self->result);
+    self->result = result;
+    Py_INCREF(self->result);
+    return cdefer_Deferred__runCallbacks(self);
+}
+
+static char cdefer_Deferred_callback_doc[] = "Run all success callbacks that have been added to this Deferred.\n\nEach callback will have its result passed as the first\nargument to the next; this way, the callbacks act as a\n\'processing chain\'. Also, if the success-callback returns a Failure\nor raises an Exception, processing will continue on the *error*-\ncallback chain.";
+
+static PyObject *cdefer_Deferred_callback(cdefer_Deferred *self, PyObject *args,
+        PyObject *kwargs) {
+    PyObject *result;
+    static char *argnames[] = {"result", NULL};
+    if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O", argnames, &result)) {
+        return NULL;
+    }
+    return cdefer_Deferred__startRunCallbacks(self, result);
+}
+
+static char cdefer_Deferred_errback_doc[] = "Run all error callbacks that have been added to this Deferred.\n\nEach callback will have its result passed as the first\nargument to the next; this way, the callbacks act as a\n\'processing chain\'. Also, if the error-callback returns a non-Failure\nor doesn\'t raise an Exception, processing will continue on the\n*success*-callback chain.\n\nIf the argument that\'s passed to me is not a Failure instance,\nit will be embedded in one. If no argument is passed, a Failure\ninstance will be created based on the current traceback stack.\n\nPassing a string as `fail\' is deprecated, and will be punished with\na warning message.";
+
+static PyObject *cdefer_Deferred_errback(cdefer_Deferred *self, PyObject *args,
+        PyObject *kwargs) {
+    PyObject *fail;
+    PyObject *tpl;
+    PyObject *tmp;
+    PyObject *result;
+    static char *argnames[] = {"fail", NULL};
+    fail = Py_None;
+    if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O", argnames, &fail)) {
+        return NULL;
+    }
+
+    if (PyObject_IsInstance(fail, failure_class)) {
+        /* Make "fail" belong to us even if we don't create a Failure
+         * wrapper (If we do, the wrapper belongs to us) */
+        Py_INCREF(fail);
+    } else {
+        tpl = Py_BuildValue("(O)", fail);
+        if (!tpl) {
+            return NULL;
+        }
+        tmp = PyObject_CallObject(failure_class, tpl);
+        Py_DECREF(tpl);
+        if (!tmp) {
+            return NULL;
+        }
+        fail = tmp;
+    }
+    result = cdefer_Deferred__startRunCallbacks(self, fail);
+    Py_DECREF(fail);
+    return result;
+}
+
+static PyObject *cdefer_Deferred__continue(cdefer_Deferred *self,
+        PyObject *args, PyObject *kwargs) {
+    PyObject *result;
+    static char *argnames[] = {"result", NULL};
+    if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O", argnames, &result)) {
+        return NULL;
+    }
+    Py_XDECREF(self->result);
+    self->result = result;
+    Py_INCREF(self->result);
+    return PyObject_CallMethod((PyObject *)self, "unpause", NULL);
+}
+
+static struct PyMethodDef cdefer_Deferred_methods[] = {
+  {"addCallbacks", (PyCFunction)cdefer_Deferred_addCallbacks,
+                   METH_VARARGS|METH_KEYWORDS, cdefer_Deferred_addCallbacks_doc},
+  {"addCallback", (PyCFunction)cdefer_Deferred_addCallback,
+                  METH_VARARGS|METH_KEYWORDS, cdefer_Deferred_addCallback_doc},
+  {"addErrback", (PyCFunction)cdefer_Deferred_addErrback,
+                 METH_VARARGS|METH_KEYWORDS, cdefer_Deferred_addErrback_doc},
+  {"addBoth", (PyCFunction)cdefer_Deferred_addBoth,
+               METH_VARARGS|METH_KEYWORDS, cdefer_Deferred_addBoth_doc},
+  {"chainDeferred", (PyCFunction)cdefer_Deferred_chainDeferred,
+                    METH_VARARGS|METH_KEYWORDS, cdefer_Deferred_chainDeferred_doc},
+  {"callback", (PyCFunction)cdefer_Deferred_callback,
+               METH_VARARGS|METH_KEYWORDS, cdefer_Deferred_callback_doc},
+  {"errback", (PyCFunction)cdefer_Deferred_errback,
+              METH_VARARGS|METH_KEYWORDS, cdefer_Deferred_errback_doc},
+  {"pause", (PyCFunction)cdefer_Deferred_pause,
+            METH_VARARGS, cdefer_Deferred_pause_doc},
+  {"unpause", (PyCFunction)cdefer_Deferred_unpause,
+              METH_VARARGS, cdefer_Deferred_unpause_doc},
+  {"_continue", (PyCFunction)cdefer_Deferred__continue,
+                METH_VARARGS|METH_KEYWORDS, ""},
+  {0, 0, 0, 0}
+};
+
+static struct PyMemberDef cdefer_Deferred_members[] = {
+  {"result", T_OBJECT_EX, offsetof(cdefer_Deferred, result), 0, 0},
+  {"paused", T_INT, offsetof(cdefer_Deferred, paused), READONLY, 0},
+  {"called", T_INT, offsetof(cdefer_Deferred, called), READONLY, 0},
+  {"callbacks", T_OBJECT, offsetof(cdefer_Deferred, callbacks), READONLY, 0},
+  {0, 0, 0, 0, 0}
+};
+
+static PyTypeObject cdefer_DeferredType = {
+    PyObject_HEAD_INIT(0)
+    0,                          /*ob_size*/
+    "cdefer.Deferred",          /*tp_name*/
+    sizeof(cdefer_Deferred),    /*tp_basicsize*/
+    0,                          /*tp_itemsize*/
+    (destructor)cdefer_Deferred_dealloc,    /*tp_dealloc*/
+    0,                          /*tp_print*/
+    0,                          /*tp_getattr*/
+    0,                          /*tp_setattr*/
+    0,                          /*tp_compare*/
+    0,                          /*tp_repr*/
+    0,                          /*tp_as_number*/
+    0,                          /*tp_as_sequence*/
+    0,                          /*tp_as_mapping*/
+    0,                          /*tp_hash */
+    0,                          /*tp_call*/
+    0,                          /*tp_str*/
+    0,                          /*tp_getattro*/
+    0,                          /*tp_setattro*/
+    0,                          /*tp_as_buffer*/
+    Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, /*tp_flags*/
+    "This is a callback which will be put off until later.\n\nWhy do we want this? Well, in cases where a function in a threaded\nprogram would block until it gets a result, for Twisted it should\nnot block. Instead, it should return a Deferred.\n\nThis can be implemented for protocols that run over the network by\nwriting an asynchronous protocol for twisted.internet. For methods\nthat come from outside packages that are not under our control, we use\nthreads (see for example L{twisted.enterprise.adbapi}).\n\nFor more information about Deferreds, see doc/howto/defer.html or\nU{http://www.twistedmatrix.com/documents/howto/defer}.", /*tp_doc*/
+    (traverseproc)cdefer_Deferred_traverse,   /*tp_traverse*/
+    (inquiry)cdefer_Deferred_clear,           /*tp_clear*/
+    0,                          /*tp_richcompare*/
+    0,                          /*tp_weaklistoffset*/
+    0,                          /*tp_iter*/
+    0,                          /*tp_iternext*/
+    cdefer_Deferred_methods,    /*tp_methods*/
+    cdefer_Deferred_members,    /*tp_members*/
+    0,                          /*tp_getset*/
+    0,                          /*tp_base*/
+    0,                          /*tp_dict*/
+    0,                          /*tp_descr_get*/
+    0,                          /*tp_descr_set*/
+    0,                          /*tp_dictoffset*/
+    (initproc)cdefer_Deferred___init__,   /*tp_init*/
+    0,                          /*tp_alloc*/
+    cdefer_Deferred_new,        /*tp_new*/
+    PyObject_GC_Del,            /*tp_free*/
+    0,                          /*tp_is_gc*/
+    0,                          /*tp_bases*/
+    0,                          /*tp_mro*/
+    0,                          /*tp_cache*/
+    0,                          /*tp_subclasses*/
+    0,                          /*tp_weaklist*/
+};
+
+static PyMethodDef cdefer_methods[] = {
+    {NULL}  /* Sentinel */
+};
+
+#ifndef PyMODINIT_FUNC  /* declarations for DLL import/export */
+#define PyMODINIT_FUNC void
+#endif
+PyMODINIT_FUNC initcdefer(void) {
+    PyObject * m = NULL;
+    PyObject * f = NULL;
+    PyObject * d = NULL;
+
+    if (PyType_Ready(&cdefer_DeferredType) < 0) {
+        return;
+    }
+
+    m = Py_InitModule3("cdefer", cdefer_methods,
+                       "cdefer");
+
+    if (!m) {
+        return;
+    }
+
+    Py_INCREF(&cdefer_DeferredType);
+    PyModule_AddObject(m, "Deferred", (PyObject *)&cdefer_DeferredType);
+
+    f = PyImport_ImportModule("twisted.python.failure");
+    if (!f) {
+        goto Error;
+    }
+
+    failure_class = PyObject_GetAttrString(f, "Failure");
+    if (!failure_class) {
+        goto Error;
+    }
+
+    d = PyImport_ImportModule("twisted.internet.defer");
+    if (!d) {
+        goto Error;
+    }
+    already_called = PyObject_GetAttrString(d, "AlreadyCalledError");
+    if (!already_called) {
+        goto Error;
+    }
+
+    debuginfo_class = PyObject_GetAttrString(d, "DebugInfo");
+    if(!debuginfo_class) {
+        goto Error;
+    }
+    
+    return;
+Error:
+    Py_XDECREF(f);
+    Py_XDECREF(failure_class);
+    Py_XDECREF(d);
+    Py_XDECREF(already_called);
+    Py_XDECREF(debuginfo_class);
+}
+

twisted/internet/defer.py

@@ -438 +438 @@
                     self.index == other.index)
         return False
 
+try:
+    from twisted.internet.cdefer import Deferred
+except ImportError:
+    pass
+else:
+    # Stubs for setDebugging/getDebugging for backwards compatibility
+    # when using cdefer.
+    is_debug = False
+    def setDebugging(new_value):
+        global is_debug
+        is_debug = new_value
+    def getDebugging():
+        global is_debug
+        return is_debug
+
 class DeferredList(Deferred):
     """I combine a group of deferreds into one callback.
 

doc/core/benchmarks/timer.py

@@ -11 +11 @@
     calls func iter times with args and kwargs, returns time elapsed
     """
 
-    import time
+    from time import time as current_time
     r = range(iter)
-    t = time.time()
+    t = current_time()
     for i in r:
         func(*args, **kwargs)
-    return time.time() - t
+    return current_time() - t

doc/core/benchmarks/defer.py

@@ +1 @@
+import functools
+from twisted.internet import defer
+from timer import timeit
+
+benchmark_funcs = []
+
+def benchmark_func(iter, args=()):
+    def decorator(func):
+        benchmark_funcs.append((func, args, iter))
+        return func
+    return decorator
+
+def benchmark_n_func(iter, ns):
+    def decorator(func):
+        for n in ns:
+            benchmark_funcs.append((func, (n,), iter))
+        return func
+    return decorator
+
+def instantiate():
+    d = defer.Deferred()
+instantiate = benchmark_func(100000)(instantiate)
+
+def instantiate_shoot_callback():
+    d = defer.Deferred()
+    d.callback(1)
+instantiate_shoot_callback = benchmark_func(100000)(instantiate_shoot_callback)
+
+def instantiate_shoot_errback():
+    d = defer.Deferred()
+    try:
+        1/0
+    except:
+        d.errback()
+    d.addErrback(lambda x: None)
+instantiate_shoot_errback = benchmark_func(200)(instantiate_shoot_errback)
+
+ns = [10, 1000, 10000]
+
+def instantiate_add_callbacks_no_result(n):
+    d = defer.Deferred()
+    def f(result):
+        return result
+    for i in xrange(n):
+        d.addCallback(f)
+        d.addErrback(f)
+        d.addBoth(f)
+        d.addCallbacks(f)
+instantiate_add_callbacks_no_result = benchmark_n_func(20, ns)(instantiate_add_callbacks_no_result)
+
+def instantiate_add_callbacks_before_result(n):
+    d = defer.Deferred()
+    def f(result):
+        return result
+    for i in xrange(n):
+        d.addCallback(f)
+        d.addErrback(f)
+        d.addBoth(f)
+        d.addCallbacks(f)
+    d.callback(1)
+instantiate_add_callbacks_before_result = benchmark_n_func(20, ns)(instantiate_add_callbacks_before_result)
+
+def instantiate_add_callbacks_after_result(n):
+    d = defer.Deferred()
+    def f(result):
+        return result
+    d.callback(1)
+    for i in xrange(n):
+        d.addCallback(f)
+        d.addErrback(f)
+        d.addBoth(f)
+        d.addCallbacks(f)
+instantiate_add_callbacks_after_result = benchmark_n_func(20, ns)(instantiate_add_callbacks_after_result)
+
+def pause_unpause(n):
+    d = defer.Deferred()
+    def f(result):
+        return result
+    d.callback(1)
+    d.pause()
+    for i in xrange(n):
+        d.addCallback(f)
+        d.addErrback(f)
+        d.addBoth(f)
+        d.addCallbacks(f)
+    d.unpause()
+pause_unpause = benchmark_n_func(20, ns)(pause_unpause)
+
+print defer.Deferred.__module__
+for func, args, iter in benchmark_funcs:
+    print func, args, timeit(func, iter, *args)