Ticket #1930: libevent-gc.patch

File libevent-gc.patch, 17.0 KB (added by antoine, 8 years ago)
  • twisted/python/_libevent.c

    diff -r 0ec4afc24de5 twisted/python/_libevent.c
    a b typedef struct EventBaseObject { 
    5454typedef struct EventBaseObject {
    5555    PyObject_HEAD
    5656    struct event_base *ev_base;
     57    /* A list of objects to be expired. We keep them here until the loop
     58       returns, otherwise it may crash libevent (who knows). */
     59    PyObject *expiredEvents;
     60    /* A dict of EventObject => None */
     61    PyObject *registeredEvents;
    5762} EventBaseObject;
    5863
    5964/* Forward declaration of CPython type object */
    typedef struct EventObject { 
    6772    struct event ev;
    6873    EventBaseObject *eventBase;
    6974    PyObject *callback;
     75    /* Duplicate original event flags since they seem to be modified when we
     76       arrive in the callback thunk */
     77    short flags;
    7078} EventObject;
    7179
    7280/* Forward declaration of CPython type object */
    static PyObject *EventBase_New(PyTypeObj 
    99107            return NULL;
    100108        }
    101109    }
     110    self->expiredEvents = PyList_New(0);
     111    self->registeredEvents = PyDict_New();
    102112    return (PyObject *)self;
    103113}
    104114
    static int EventBase_Init(EventBaseObjec 
    127137    return 0;
    128138}
    129139
     140/* Internal helper, destroy expired events */
     141static void EventBase_ExpireEvents(EventBaseObject *self) {
     142    /* Destroy and recreate an empty list, so as to to decref all events
     143       as well as their contents */
     144    Py_CLEAR(self->expiredEvents);
     145    self->expiredEvents = PyList_New(0);
     146}
     147
     148/* Internal helper, register an event */
     149static void EventBase_RegisterEvent(EventBaseObject *self, PyObject *obj) {
     150    PyDict_SetItem(self->registeredEvents, obj, Py_None);
     151}
     152
     153/* Internal helper, unregister an event */
     154static void EventBase_UnregisterEvent(EventBaseObject *self, PyObject *obj) {
     155    PyDict_DelItem(self->registeredEvents, obj);
     156}
     157
     158
     159
     160static int
     161EventBase_Traverse(EventBaseObject *self, visitproc visit, void *arg)
     162{
     163    int vret;
     164
     165    if (self->registeredEvents) {
     166        vret = visit(self->registeredEvents, arg);
     167        if (vret != 0)
     168            return vret;
     169    }
     170    if (self->expiredEvents) {
     171        vret = visit(self->expiredEvents, arg);
     172        if (vret != 0)
     173            return vret;
     174    }
     175
     176    return 0;
     177}
     178
     179static int
     180EventBase_Clear(EventBaseObject *self)
     181{
     182    Py_CLEAR(self->registeredEvents);
     183    Py_CLEAR(self->expiredEvents);
     184    return 0;
     185}
     186
    130187/* EventBaseObject destructor */
    131188static void EventBase_Dealloc(EventBaseObject *obj) {
     189    EventBase_Clear(obj);
     190    /* Unfortunately the following line can make libevent crash (w/ 1.4.1-beta)
     191       ("event.c:241: event_base_free: Assertion `min_heap_empty(&base->timeheap)' failed.")
     192    */
     193//     event_base_free(obj->ev_base);
    132194    obj->ob_type->tp_free((PyObject *)obj);
    133195}
     196
    134197
    135198/* EventBaseObject methods */
    136199PyDoc_STRVAR(EventBase_LoopDoc,
    static PyObject *EventBase_Loop(EventBas 
    152215    rv = event_base_loop(self->ev_base, flags);
    153216    Py_END_ALLOW_THREADS
    154217
     218    EventBase_ExpireEvents(self);
    155219    if (PyErr_Occurred()) {
    156220        return NULL;
    157221    }
    static PyObject *EventBase_LoopExit(Even 
    177241
    178242    tv.tv_sec = (long) exitAfterSecs;
    179243    tv.tv_usec = (exitAfterSecs - (long) exitAfterSecs) * 1000000;
     244    Py_BEGIN_ALLOW_THREADS
    180245    rv = event_base_loopexit(self->ev_base, &tv);
     246    Py_END_ALLOW_THREADS
     247
     248    EventBase_ExpireEvents(self);
     249    if (PyErr_Occurred()) {
     250        return NULL;
     251    }
     252
    181253    return PyInt_FromLong(rv);
    182254}
    183255
    explicit call to EventBase.loopExit() or 
    189261explicit call to EventBase.loopExit() or via a signal, or if a callback \n\
    190262raises an exception.");
    191263static PyObject *EventBase_Dispatch(EventBaseObject *self) {
    192 
    193     int rv = event_base_dispatch(self->ev_base);
     264    int rv = 0;
     265    Py_BEGIN_ALLOW_THREADS
     266    rv = event_base_dispatch(self->ev_base);
     267    Py_END_ALLOW_THREADS
     268    EventBase_ExpireEvents(self);
    194269    if (PyErr_Occurred()) {
    195270        return NULL;
    196271    }
    static PyTypeObject EventBase_Type = { 
    335410    PyObject_GenericGetAttr,                   /*tp_getattro*/
    336411    PyObject_GenericSetAttr,                   /*tp_setattro*/
    337412    0,                                         /*tp_as_buffer*/
    338     Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/
     413    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, /*tp_flags*/
    339414    0,                                         /*tp_doc*/
    340     0,                                         /*tp_traverse*/
    341     0,                                         /*tp_clear*/
     415    (traverseproc)EventBase_Traverse,          /*tp_traverse*/
     416    (inquiry)EventBase_Clear,                  /*tp_clear*/
    342417    0,                                         /*tp_richcompare*/
    343418    0,                                         /*tp_weaklistoffset*/
    344419    0,                                         /*tp_iter*/
    static PyTypeObject EventBase_Type = { 
    353428    0,                                         /*tp_dictoffset*/
    354429    (initproc)EventBase_Init,                  /*tp_init*/
    355430    PyType_GenericAlloc,                       /*tp_alloc*/
    356     EventBase_New,                             /*tp_new*/
    357     PyObject_Del,                              /*tp_free*/
    358     0                                          /*tp_is_gc*/
     431    EventBase_New                              /*tp_new*/
    359432};
    360433
    361434/* Typechecker */
    static void __libevent_ev_callback(int f 
    376449static void __libevent_ev_callback(int fd, short events, void *arg) {
    377450    EventObject *ev = arg;
    378451    PyObject *result = 0;
    379     PyObject *tupleArgs = PyTuple_New(3);
    380     Py_INCREF((PyObject *) ev);
    381     Py_INCREF((PyObject *) ev);
    382     PyTuple_SET_ITEM(tupleArgs, 0, PyInt_FromLong(fd));
    383     PyTuple_SET_ITEM(tupleArgs, 1, PyInt_FromLong(events));
    384     PyTuple_SET_ITEM(tupleArgs, 2, (PyObject *) ev);
     452    PyObject *tupleArgs;
    385453    PyGILState_STATE gstate;
    386454    gstate = PyGILState_Ensure();
     455    tupleArgs = Py_BuildValue("(iiO)", fd, events, ev);
     456    Py_INCREF((PyObject *) ev);
    387457    result = PyObject_CallObject(ev->callback, tupleArgs);
    388     Py_DECREF((PyObject *) ev);
    389458    Py_CLEAR(tupleArgs);
     459    if (!(ev->flags & EV_PERSIST)) {
     460        /* Register the event for deletion but do not delete it right now.
     461           The list will be destroyed and its contents deallocated when the
     462           event loop returns. */
     463        PyList_Append(ev->eventBase->expiredEvents, (PyObject *) ev);
     464        EventBase_UnregisterEvent(ev->eventBase, (PyObject *) ev);
     465    }
    390466    if (result) {
    391467        Py_CLEAR(result);
    392468    }
    static void __libevent_ev_callback(int f 
    397473        /* Exit the loop, so that the error pops out to dispatch/loop. */
    398474        event_base_loopexit(ev->ev.ev_base, &tv);
    399475    }
     476    Py_DECREF((PyObject *) ev);
    400477    Safe_PyGILState_Release(gstate);
    401478}
    402479
    static int Event_Init(EventObject *self, 
    430507
    431508    Py_INCREF(callback);
    432509    self->callback = callback;
     510    self->flags = events;
    433511    return 0;
    434512}
     513
     514static int
     515Event_Traverse(EventObject *self, visitproc visit, void *arg)
     516{
     517    int vret;
     518
     519    if (self->eventBase) {
     520        vret = visit((PyObject *) self->eventBase, arg);
     521        if (vret != 0)
     522            return vret;
     523    }
     524    if (self->callback) {
     525        vret = visit(self->callback, arg);
     526        if (vret != 0)
     527            return vret;
     528    }
     529
     530    return 0;
     531}
     532
     533static int
     534Event_Clear(EventObject *self)
     535{
     536    Py_CLEAR(self->callback);
     537    Py_CLEAR(self->eventBase);
     538    return 0;
     539}
     540
    435541
    436542PyDoc_STRVAR(Event_SetPriorityDoc,
    437543"setPriority(self, priority)\n\
    static PyObject *Event_SetPriority(Event 
    451557                "error setting event priority - event is either already active or priorities are not enabled");
    452558        return NULL;
    453559    }
    454     Py_INCREF(Py_None);
    455     return Py_None;
     560    Py_RETURN_NONE;
    456561}
    457562
    458563PyDoc_STRVAR(Event_AddToLoopDoc,
    static PyObject *Event_AddToLoop(EventOb 
    476581    if (timeout >= 0.0) {
    477582        tv.tv_sec = (long) timeout;
    478583        tv.tv_usec = (timeout - (long) timeout) * 1000000;
    479         rv = event_add(&((EventObject *) self)->ev, &tv);
     584        rv = event_add(&self->ev, &tv);
    480585    }
    481586    else {
    482         rv = event_add(&((EventObject *) self)->ev, NULL);
     587        rv = event_add(&self->ev, NULL);
    483588    }
    484589    if (rv != 0) {
    485590        return PyErr_SetFromErrno(EventErrorObject);
    486591    }
    487     Py_INCREF(self);
    488     Py_INCREF(Py_None);
    489     return Py_None;
     592    EventBase_RegisterEvent(self->eventBase, (PyObject *) self);
     593    Py_RETURN_NONE;
    490594}
    491595
    492596PyDoc_STRVAR(Event_RemoveFromLoopDoc,
    static PyObject *Event_RemoveFromLoop(Ev 
    498602    if (event_del(&self->ev) < 0) {
    499603        return PyErr_SetFromErrno(EventErrorObject);
    500604    }
    501     Py_DECREF(self);
    502     Py_INCREF(Py_None);
    503     return Py_None;
     605    EventBase_UnregisterEvent(self->eventBase, (PyObject *) self);
     606    Py_RETURN_NONE;
    504607}
    505608
    506609PyDoc_STRVAR(Event_SetEventBaseDoc,
    static PyObject *Event_SetEventBase(Even 
    510613static PyObject *Event_SetEventBase(EventObject *self, PyObject *args,
    511614                    PyObject *kwargs) {
    512615    static char *kwlist[] = {"eventBase", NULL};
    513     PyObject *eventBase;
     616    PyObject *eventBase, *old_eventBase;
    514617    int rv = 0;
    515618
    516619    if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O", kwlist, &eventBase)) {
    static PyObject *Event_SetEventBase(Even 
    527630        return NULL;
    528631    }
    529632    Py_INCREF(eventBase);
    530     Py_XDECREF(self->eventBase);
     633    /* Thread-safe way of removing an attr value */
     634    old_eventBase = (PyObject *) self->eventBase;
    531635    self->eventBase = (EventBaseObject *)eventBase;
    532     Py_INCREF(Py_None);
    533     return Py_None;
     636    Py_XDECREF(old_eventBase);
     637    Py_RETURN_NONE;
    534638}
    535639
    536640PyDoc_STRVAR(Event_PendingDoc,
    static PyObject *Event_GetTimeout(EventO 
    559663        d = tv.tv_sec + (tv.tv_usec / 1000000.0);
    560664        return PyFloat_FromDouble(d);
    561665    }
    562     Py_INCREF(Py_None);
    563     return Py_None;
     666    Py_RETURN_NONE;
    564667}
    565668
    566669PyDoc_STRVAR(Event_FilenoDoc,
    static PyObject *Event_Fileno(EventObjec 
    574677
    575678/* EventObject destructor */
    576679static void Event_Dealloc(EventObject *obj) {
    577     Py_XDECREF(obj->eventBase);
    578     Py_XDECREF(obj->callback);
     680    Event_Clear(obj);
    579681    obj->ob_type->tp_free((PyObject *)obj);
    580682}
    581683
    static PyTypeObject Event_Type = { 
    650752    PyObject_GenericGetAttr,                   /*tp_getattro*/
    651753    PyObject_GenericSetAttr,                   /*tp_setattro*/
    652754    0,                                         /*tp_as_buffer*/
    653     Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,  /*tp_flags*/
     755    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,  /*tp_flags*/
    654756    0,                                         /*tp_doc*/
    655     0,                                         /*tp_traverse*/
    656     0,                                         /*tp_clear*/
     757    (traverseproc)Event_Traverse,              /*tp_traverse*/
     758    (inquiry)Event_Clear,                      /*tp_clear*/
    657759    0,                                         /*tp_richcompare*/
    658760    0,                                         /*tp_weaklistoffset*/
    659761    0,                                         /*tp_iter*/
    static PyTypeObject Event_Type = { 
    668770    0,                                         /*tp_dictoffset*/
    669771    (initproc)Event_Init,                      /*tp_init*/
    670772    PyType_GenericAlloc,                       /*tp_alloc*/
    671     Event_New,                                 /*tp_new*/
    672     PyObject_Del,                              /*tp_free*/
    673     0,                                         /*tp_is_gc*/
     773    Event_New                                  /*tp_new*/
    674774};
    675775
    676776static PyMethodDef EventModule_Functions[] = {
  • twisted/test/test_libevent.py

    diff -r 0ec4afc24de5 twisted/test/test_libevent.py
    a b  
    1 # Copyright (c) 2007-2008 Twisted Matrix Laboratories.
     1# Copyright (c) 2007 Twisted Matrix Laboratories.
    22# See LICENSE for details.
    33
    44"""
    5 Tests for L{libevent} wrapper.
     5Tests for libevent wrapper.
    66"""
    77
    8 import socket, errno, sys, weakref, gc
     8import socket, errno, sys, os, weakref, gc
    99
    1010from twisted.trial import unittest
    1111
    class ConnectedEventTestCase(unittest.Te 
    148148            client.connect(('127.0.0.1', self.serverSocket.getsockname()[1]))
    149149        except socket.error, e:
    150150            self.assertEquals(e.args[0], errno.EINPROGRESS)
     151        else:
     152            raise unittest.FailTest("Connect should have raised EINPROGRESS")
    151153        server, addr = self.serverSocket.accept()
    152154
    153155        self.connections.extend((client, server))
    class EventBaseTestCase(unittest.TestCas 
    342344        timer.addToLoop(0.01)
    343345        self.assertRaises(RuntimeError, newEventBase.loop, libevent.EVLOOP_ONCE)
    344346
    345 
    346347    def test_dispatchError(self):
    347348        """
    348349        Check that dispatch forwards exception raised in callback.
    class EventBaseTestCase(unittest.TestCas 
    367368        newEventBase.dispatch()
    368369        self.assertEquals(len(fireEvents), 1)
    369370
    370 
    371371    def test_successfulCallbackReference(self):
    372372        """
    373373        Check that successful callbacks aren't leaked.
    class EventBaseTestCase(unittest.TestCas 
    375375        newEventBase = libevent.EventBase()
    376376        def cb(fd, events, obj):
    377377            pass
    378         import sys
    379378        self._watchForLeaks(cb)
    380379        timer = newEventBase.createTimer(cb)
    381380        timer.addToLoop(0.002)
    class EventBaseTestCase(unittest.TestCas 
    383382
    384383        del cb, timer
    385384        self._assertLeaks()
    386 
    387385
    388386    def test_failedCallbackReference(self):
    389387        """
    class EventBaseTestCase(unittest.TestCas 
    400398        del eb, timer
    401399        self._assertLeaks()
    402400
    403 
    404401    def test_unfiredCallbackReference(self):
    405402        """
    406403        Check that unfired callbacks aren't leaked when the eventBase is
    class EventBaseTestCase(unittest.TestCas 
    410407        def cb(fd, events, obj):
    411408            pass
    412409        self._watchForLeaks(cb)
     410        #print map(sys.getrefcount, [newEventBase, cb])
    413411        timer = newEventBase.createTimer(cb)
     412        #print map(sys.getrefcount, [newEventBase, timer, cb])
    414413        timer.addToLoop(1)
     414        #print map(sys.getrefcount, [newEventBase, timer, cb])
    415415
    416416        del cb, timer, newEventBase
    417417        self._assertLeaks()
    418 
    419418
    420419    def test_callbackReference(self):
    421420        """
    class EventBaseTestCase(unittest.TestCas 
    429428
    430429        del cb, timer
    431430        self._assertLeaks()
    432 
    433431
    434432    def test_callbackExceptionReference(self):
    435433        """
    class EventBaseTestCase(unittest.TestCas 
    449447        self._watchForLeaks(exc[0])
    450448
    451449        del exc[0]
    452 
    453450        self._assertLeaks()
    454 
    455451
    456452    def test_callbackSurvival(self):
    457453        """
    class EventBaseTestCase(unittest.TestCas 
    459455        dies.
    460456        """
    461457        newEventBase = libevent.EventBase()
    462         fireEvents = []
    463458        def cb(fd, events, obj):
    464             fireEvents.append((fd, events, obj))
     459            pass
    465460        timer = newEventBase.createTimer(cb)
    466461        timer.addToLoop(1)
    467462        self._watchForSurvival(cb)
    class EventBaseTestCase(unittest.TestCas 
    469464        del cb, timer
    470465        self._assertSurvival()
    471466
     467    def test_persistentCallbackSurvival(self):
     468        """
     469        Check that a persistent callback survives after been fired.
     470        """
     471        rfd, wfd = os.pipe()
     472        newEventBase = libevent.EventBase()
     473        def cb(fd, events, obj):
     474            newEventBase.loopExit(0)
     475        timer = newEventBase.createEvent(rfd,
     476            libevent.EV_READ | libevent.EV_PERSIST, cb)
     477        timer.addToLoop()
     478        os.write(wfd, " ")
     479        newEventBase.dispatch()
     480        self._watchForSurvival(cb)
     481
     482        del cb, timer
     483        self._assertSurvival()
     484
     485    def test_persistentFailedCallbackSurvival(self):
     486        """
     487        Check that a persistent callback survives after raising an exception.
     488        """
     489        rfd, wfd = os.pipe()
     490        newEventBase = libevent.EventBase()
     491        def cb(fd, events, obj):
     492            newEventBase.loopExit(0)
     493            raise RuntimeError("foo")
     494        timer = newEventBase.createEvent(rfd,
     495            libevent.EV_READ | libevent.EV_PERSIST, cb)
     496        timer.addToLoop()
     497        os.write(wfd, " ")
     498        self.assertRaises(RuntimeError, newEventBase.dispatch)
     499        self._watchForSurvival(cb)
     500
     501        del cb, timer
     502        self._assertSurvival()
     503
     504    def test_persistentCallbackReference(self):
     505        """
     506        Check that a persistent callback doesn't leak when the eventBase
     507        is destroyed.
     508        """
     509        rfd, wfd = os.pipe()
     510        newEventBase = libevent.EventBase()
     511        def cb(fd, events, obj):
     512            newEventBase.loopExit(0)
     513        timer = newEventBase.createEvent(rfd,
     514            libevent.EV_READ | libevent.EV_PERSIST, cb)
     515        timer.addToLoop()
     516        os.write(wfd, " ")
     517        newEventBase.dispatch()
     518        self._watchForLeaks(cb)
     519
     520        newEventBase = None
     521        del cb, timer
     522        self._assertLeaks()
    472523
    473524    def test_dispatchedEventRefCount(self):
    474525        """