[Twisted-web] Fix for an issue 92 (windows binary distribution
made with distutils doesn't install data files into
package directories)
Cory Dodt
corydodt at twistedmatrix.com
Thu Mar 3 10:33:42 MST 2005
-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1
+1 performance-enhancing drugs.
BTW, is there an example of using cached anywhere?
C
Andrea Arcangeli wrote:
| On Wed, Mar 02, 2005 at 10:48:03PM +0000, Matt Goodall wrote:
|
|>Yes, someone will.
|
|
| btw, I would also like to send a reminder about getting the caching
| stuff into the trunk ;). I proved that the hard approach in rend.Page is
| significantly more performant than the tags.cached pure approach (even
| the hard approach is less clean).
|
| This is what I carry in my own Nevow branch (most of it is dialtone's
| Nevow-caching branch):
|
| Index: Nevow/nevow/tags.py
| ===================================================================
| --- Nevow/nevow/tags.py (revision 1257)
| +++ Nevow/nevow/tags.py (working copy)
| @@ -25,7 +25,7 @@
| """
|
|
| -from nevow.stan import Proto, Tag, directive, raw, xml, CommentProto,
invisible, slot, cdata
| +from nevow.stan import Proto, Tag, directive, raw, xml, CommentProto,
invisible, slot, cdata, cached
|
|
| comment = CommentProto()
| @@ -62,7 +62,9 @@
| def inlineJS(s):
| return script(type="text/javascript",
language="JavaScript")[xml('\n//<![CDATA[\n%s\n//]]>\n' % s)]
|
| -__all__ = tags + ['invisible', 'comment', '_dir', '_del', '_object',
'_map', 'drange', 'Tag', 'directive', 'xml', 'raw', 'slot', 'cdata',
'inlineJS'] + ['_%s' % x for x in range(100)]
| +__all__ = tags + ['invisible', 'comment', '_dir', '_del', '_object',
| + '_map', 'drange', 'Tag', 'directive', 'xml', 'raw',
| + 'slot', 'cached', 'cdata', 'inlineJS'] + ['_%s' % x for x
in range(100)]
|
|
| ########################
| Index: Nevow/nevow/__init__.py
| ===================================================================
| --- Nevow/nevow/__init__.py (revision 1257)
| +++ Nevow/nevow/__init__.py (working copy)
| @@ -138,6 +138,8 @@
| nevow.util.remainingSegmentsFactory nevow.context.RequestContext
nevow.inevow.IRemainingSegments
| nevow.util.currentSegmentsFactory nevow.context.RequestContext
nevow.inevow.ICurrentSegments
|
| +nevow.cache.SiteCache nevow.context.SiteContext nevow.inevow.ICache
| +
| nevow.query.QueryContext nevow.context.WovenContext nevow.inevow.IQ
| nevow.query.QueryLoader nevow.inevow.IDocFactory nevow.inevow.IQ
| nevow.query.QueryList __builtin__.list nevow.inevow.IQ
| @@ -186,6 +188,7 @@
| nevow.flat.flatstan.RendererSerializer nevow.inevow.IRenderer
| nevow.flat.flatstan.DirectiveSerializer nevow.stan.directive
| nevow.flat.flatstan.SlotSerializer nevow.stan.slot
| +nevow.flat.flatstan.CachedSerializer nevow.stan.cached
| nevow.flat.flatstan.ContextSerializer nevow.context.WovenContext
| nevow.flat.flatstan.DeferredSerializer
twisted.internet.defer.Deferred
| nevow.flat.flatstan.DeferredSerializer
twisted.internet.defer.DeferredList
| Index: Nevow/nevow/flat/flatstan.py
| ===================================================================
| --- Nevow/nevow/flat/flatstan.py (revision 1257)
| +++ Nevow/nevow/flat/flatstan.py (working copy)
| @@ -8,11 +8,15 @@
|
| from nevow import util
| from nevow.stan import Proto, Tag, xml, directive, Unset, invisible
| -from nevow.inevow import IRenderer, IRendererFactory, IGettable, IData
| -from nevow.flat import precompile, serialize
| +from nevow.inevow import IRenderer, IRendererFactory, IGettable, IData, ICache
| +from nevow.flat import precompile, serialize, iterflatten
| from nevow.accessors import convertToData
| from nevow.context import WovenContext
|
| +from time import time as now
| +from cStringIO import StringIO
| +from twisted.internet import defer
| +
| allowSingleton = ('img', 'br', 'hr', 'base', 'meta', 'link', 'param', 'area',
| 'input', 'col', 'basefont', 'isindex', 'frame')
|
| @@ -226,6 +230,43 @@
| return serialize(original.default, context)
| return serialize(data, context)
|
| +def CachedSerializer(original, context):
| + cache = ICache(original.scope(context))
| + cached = cache.get(original.key, original.lifetime)
| + if cached:
| + yield cached
| + return
| + io = StringIO()
| + for child in iterflatten(original.children, context, io.write,
| + lambda item: True):
| + if isinstance(child, tuple):
| + childDeferred, childReturner = child
| +
| + d = defer.Deferred() ## A new deferred for the outer loop,
whose result
| + ## we don't care about, because we don't want the outer loop to
write
| + ## anything when this deferred fires -- only when the entire
for loop
| + ## has completed and we have all the "children" flattened
| +
| + def innerDeferredResultAvailable(result):
| + childReturner(result) ## Cause the inner iterflatten to
continue
| + d.callback('') ## Cause the outer iterflatten to continue
| + return ''
| +
| + childDeferred.addCallback(innerDeferredResultAvailable)
| +
| + ## Make the outer loop wait on our new deferred.
| + ## We call the new deferred back with ''
| + ## Which will cause the outer loop to write '' to the request,
| + ## which doesn't matter. It will then call our "returner",
| + ## which is just the noop lambda below, because we don't care
| + ## about the return result of the new deferred, which is just
| + ## ''
| +
| + yield d, lambda result: ''
| + result = io.getvalue()
| + cache.set(result, original.key)
| + yield result
| +
| def ContextSerializer(original, context):
| originalContext = original.clone(deep=False)
| originalContext.precompile = context and context.precompile or False
| Index: Nevow/nevow/stan.py
| ===================================================================
| --- Nevow/nevow/stan.py (revision 1257)
| +++ Nevow/nevow/stan.py (working copy)
| @@ -119,8 +119,40 @@
| """
| raise NotImplementedError, "Stan slot instances are not iterable."
|
| +def passThrough(_):
| + return _
|
| +class cached(object):
| + """Marker for cached content
| + """
| + __slots__ = ['key', 'children', 'lifetime', 'scope']
|
| + def __init__(self, key, scope=None, lifetime=-1):
| + self.key = key
| + self.children = []
| + self.lifetime = lifetime
| + self.scope = scope
| + if not scope:
| + self.scope = passThrough
| +
| +
| + def __repr__(self):
| + return "cached('%s','%s')" % self.key, self.lifetime
| +
| + def __getitem__(self, children):
| + """cached content is what is being cached
| + """
| + if not isinstance(children, (list, tuple)):
| + children = [children]
| + self.children.extend(children)
| + return self
| +
| + def __iter__(self):
| + """Prevent an infinite loop if someone tries to do
| + for x in cached('foo'):
| + """
| + raise NotImplementedError, "Stan slot instances are not iterable."
| +
| class Tag(object):
| """Tag instances represent XML tags with a tag name, attributes,
| and children. Tag instances can be constructed using the Prototype
| Index: Nevow/nevow/inevow.py
| ===================================================================
| --- Nevow/nevow/inevow.py (revision 1257)
| +++ Nevow/nevow/inevow.py (working copy)
| @@ -98,8 +98,24 @@
|
| ANY python object is said to implement IData.
| """
| +class ICache(compy.Interface):
| + """This object represents the cache that contains all the
| + pre-flattened fragments
| + """
| + def get(self, index, lifetime):
| + """ Get an object from the cache with the given index only if
| + it is less old than lifetime, otherwise return None.
| + """
|
| + def set(self, toBeCached, *indexes):
| + """ Register toBeCached with each of the indexes passed """
|
| + def clear(self, what):
| + """ Clear what keyed element from the cache, or search for
| + what in sequences in all the keys and clear the item
| + """
| +
| +
| class IGettable(compy.Interface):
| def get(self, context):
| """Return the data
| Index: Nevow/nevow/rend.py
| ===================================================================
| --- Nevow/nevow/rend.py (revision 1257)
| +++ Nevow/nevow/rend.py (working copy)
| @@ -30,6 +30,7 @@
| from nevow import flat
| from nevow.util import log
| from nevow import util
| +from nevow import url
|
| import formless
| from formless import iformless
| @@ -376,6 +377,8 @@
| self.children[name] = child
|
|
| +_CACHE = {}
| +
| class Page(Fragment, ConfigurableFactory, ChildLookupMixin):
| """A page is the main Nevow resource and renders a document loaded
| via the document factory (docFactory).
| @@ -389,8 +392,37 @@
| afterRender = None
| addSlash = None
|
| + cache = False
| + lifetime = -1
| + __lastCacheRendering = 0
| +
| flattenFactory = flat.flattenFactory
|
| + def hasCache(self, ctx):
| + if not self.cache:
| + return
| +
| + _now = now() # run gettimeofday only once
| + timeout = _now > self.__lastCacheRendering + self.lifetime and \
| + self.lifetime >= 0
| + c = self.lookupCache(ctx)
| + if timeout or c is None:
| + # stop other renders
| + self.__lastCacheRendering = _now
| + c = None
| + return c
| + def cacheRendered(self, ctx, c):
| + if not self.cache:
| + return
| + # overwrite the deferred with the data
| + self.storeCache(ctx, c)
| + def cacheIDX(self, ctx):
| + return str(url.URL.fromContext(ctx))
| + def storeCache(self, ctx, c):
| + _CACHE[self.cacheIDX(ctx)] = c
| + def lookupCache(self, ctx):
| + return _CACHE.get(self.cacheIDX(ctx))
| +
| def renderHTTP(self, ctx):
| ## XXX request is really ctx now, change the name here
| request = inevow.IRequest(ctx)
| @@ -412,11 +444,18 @@
| if self.afterRender is not None:
| self.afterRender(ctx)
|
| - if self.buffered:
| + c = self.hasCache(ctx)
| + if c is not None:
| + finishRequest()
| + return c
| +
| + if self.buffered or self.cache:
| io = StringIO()
| writer = io.write
| def finisher(result):
| - request.write(io.getvalue())
| + c = io.getvalue()
| + self.cacheRendered(ctx, c)
| + request.write(c)
| finishRequest()
| return result
| else:
| @@ -500,7 +539,6 @@
| else:
| ## Use the redirectAfterPost url
| ref = str(redirectAfterPost)
| - from nevow import url
| refpath = url.URL.fromString(ref)
| magicCookie = str(now())
| refpath = refpath.replace('_nevow_carryover_', magicCookie)
| Index: Nevow/nevow/guard.py
| ===================================================================
| --- Nevow/nevow/guard.py (revision 1257)
| +++ Nevow/nevow/guard.py (working copy)
| @@ -24,7 +24,7 @@
| from twisted.protocols import http
|
| # Nevow imports
| -from nevow import inevow, url, stan
| +from nevow import inevow, url, stan, cache
|
|
| def _sessionCookie():
| @@ -315,6 +315,7 @@
| path="/%s" % '/'.join(request.prepath),
| secure=secure)
| sz = self.sessions[newCookie] = self.sessionFactory(self, newCookie)
| + sz.setComponent(inevow.ICache, cache.SessionCache())
| sz.args = request.args
| sz.fields = getattr(request, 'fields', {})
| sz.content = request.content
| Index: Nevow/nevow/cache.py
| ===================================================================
| --- Nevow/nevow/cache.py (revision 0)
| +++ Nevow/nevow/cache.py (revision 0)
| @@ -0,0 +1,33 @@
| +from time import time as now
| +from nevow import inevow
| +
| +class SiteCache(object):
| + __implements__ = inevow.ICache,
| + _content = {}
| + def __init__(self, original):
| + self.original = original
| +
| + def get(self, index, lifetime):
| + cached = self._content.get(index, None)
| + if cached is None:
| + return
| + if lifetime < 0:
| + return cached[1]
| + if cached[0] + lifetime > now():
| + return cached[1]
| +
| + def set(self, toBeCached, *indexes):
| + _now = now()
| + for index in indexes:
| + self._content[index] = (_now, toBeCached)
| +
| + def clear(self, what):
| + if self._content.has_key(what):
| + self._content.pop(what)
| + for key in self._content.keys():
| + if what in key:
| + self._content.pop(key)
| +
| +class SessionCache(SiteCache):
| + def __init__(self):
| + self._content = {}
|
| This code is working for a few weeks on www.cpushare.com, so far so
| good (all http part is completely cached with the rand.Page lifetime and
| it delivers >200 req per second of those small pages). I also use the
| tags.caching from dialtime in various places.
|
| Thanks a lot to dialtone and everyone else for making this possible.
|
| _______________________________________________
| Twisted-web mailing list
| Twisted-web at twistedmatrix.com
| http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-web
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v1.2.1 (MingW32)
Comment: Using GnuPG with Thunderbird - http://enigmail.mozdev.org
iD8DBQFCJ0p23A5SrXAiHQcRApmdAJ96j3+OTlRyXjb5crYzoq6xq/lDBACcCEas
F0Mrvrylx1FOipT8q0EfQ8Y=
=iRMT
-----END PGP SIGNATURE-----
More information about the Twisted-web
mailing list