[Twisted-Python] Per-connection and per-protocol stats

[Itamar Shtull-Trauring]

As I mentioned before in the metrics message, I want to have for each 
protocol events stats, per server and per server connection. This is the 
code I currently use (comments on the hits per second algorithm and in 
general are welcome.) I also have a web Resource for this so I can view 
stats with a nice interface.


======================================
"""Event monitoring support"""

import operator

from twisted.python import threadable, delay


class ProtocolMonitor:
     """Monitor a server that serves connections.

     Accepts a list of event names which the protocol can then log with
     log_event.
     """

     def __init__(self, name, eventNames):
         self.name = name
         self.eventNames = eventNames
         self.connections = {}
         self.eventcounters = EventCounters(eventNames)

     def install(self):
         """Make sure reset() is run every 0.1 seconds."""
         from twisted.internet import main
         self.looper = delay.Delayed()
         self.looper.ticktime = 0.1
         self.looper.loop(self.reset, 1)
         main.addDelayed(self.looper)

     def uninstall(self):
         """Uninstall Delayed."""
         from twisted.internet import main
         main.removeDelayed(self.looper)
         del self.looper

     def reset(self):
         """Runs once a second, resets all EventCounters."""
         for e in self.connections.values():
             e.reset()
         self.eventcounters.reset()

     def log_connectionOpened(self, conn):
         self.connections[conn] = EventCounters(self.eventNames)

     def log_connectionClosed(self, conn):
         del self.connections[conn]

     def log_event(self, conn, name):
         """First parameter should be a Protocol, second an event"""
         assert name in self.eventNames
         self.eventcounters.increment(name)
         if self.connections.has_key(conn):
             self.connections[conn].increment(name)

     synchronized = ["log_connectionOpened", "log_connectionClosed", 
"log_event", "reset"]


class EventCounters:
     """Stores a number of event counters, and how many a second.

     Algorithm for determining how many events a second:

     Divide each second into 10 (N) equal parts. Make an 11 (N+1) items long
     list L containing how many events in each part of the second, plus the
     last part of the previous second.

         L = [0] * 11

     When a new event is added, increment last item in list:

         L[-1] += 1

     Every 0.1 (1.0/N) seconds, remove first item in L and append 0:

         del L[0]; L.append(0)

     To calculate how many events in the past second, sum up last 10 (N)
     parts of L and 0.5 of the first item in L. This gives us how many hits
     in the past 0.95 to 1.05 seconds (1-1.0/2N to 1+1.0/2N seconds),
     depending on when we run this function:

         (0.5 * L[0]) + reduce(operator.add, L[1:])

     """

     def __init__(self, eventNames, ):
         self.eventNames = eventNames
         self.counters = {}
         self.secondCounters = {}
         for name in eventNames:
             self.counters[name] = 0
             self.secondCounters[name] = [0] * 10

     def increment(self, name):
         self.counters[name] += 1
         self.secondCounters[name][-1] += 1

     def reset(self):
         """Should be called every 0.1 seconds."""
         for name in self.eventNames:
             del self.secondCounters[name][0]
             self.secondCounters[name].append(0)

     def getInfo(self):
         """Return a dictionary.

         The key is an event name and the value a tuple:
         (counters, counters per second)."""
         result = {}
         for name in self.eventNames:
             l = self.secondCounters[name]
             perSecond = (0.5 * l[0]) + reduce(operator.add, l[1:])
             result[name] = (self.counters[name], perSecond)
         return result

     synchronized = ["increment", "reset", "getInfo"]


threadable.synchronize(ProtocolMonitor)
threadable.synchronize(EventCounters)