| 1 |
|
|---|
| 2 |
|
|---|
| 3 |
|
|---|
| 4 |
|
|---|
| 5 |
|
|---|
| 6 |
|
|---|
| 7 |
""" |
|---|
| 8 |
Different styles of persisted objects. |
|---|
| 9 |
""" |
|---|
| 10 |
|
|---|
| 11 |
|
|---|
| 12 |
import types |
|---|
| 13 |
import copy_reg |
|---|
| 14 |
import copy |
|---|
| 15 |
|
|---|
| 16 |
try: |
|---|
| 17 |
import cStringIO as StringIO |
|---|
| 18 |
except ImportError: |
|---|
| 19 |
import StringIO |
|---|
| 20 |
|
|---|
| 21 |
|
|---|
| 22 |
from twisted.python import log |
|---|
| 23 |
|
|---|
| 24 |
try: |
|---|
| 25 |
from new import instancemethod |
|---|
| 26 |
except: |
|---|
| 27 |
from org.python.core import PyMethod |
|---|
| 28 |
instancemethod = PyMethod |
|---|
| 29 |
|
|---|
| 30 |
oldModules = {} |
|---|
| 31 |
|
|---|
| 32 |
|
|---|
| 33 |
|
|---|
| 34 |
|
|---|
| 35 |
def pickleMethod(method): |
|---|
| 36 |
'support function for copy_reg to pickle method refs' |
|---|
| 37 |
return unpickleMethod, (method.im_func.__name__, |
|---|
| 38 |
method.im_self, |
|---|
| 39 |
method.im_class) |
|---|
| 40 |
|
|---|
| 41 |
def unpickleMethod(im_name, |
|---|
| 42 |
im_self, |
|---|
| 43 |
im_class): |
|---|
| 44 |
'support function for copy_reg to unpickle method refs' |
|---|
| 45 |
try: |
|---|
| 46 |
unbound = getattr(im_class,im_name) |
|---|
| 47 |
if im_self is None: |
|---|
| 48 |
return unbound |
|---|
| 49 |
bound=instancemethod(unbound.im_func, |
|---|
| 50 |
im_self, |
|---|
| 51 |
im_class) |
|---|
| 52 |
return bound |
|---|
| 53 |
except AttributeError: |
|---|
| 54 |
log.msg("Method",im_name,"not on class",im_class) |
|---|
| 55 |
assert im_self is not None,"No recourse: no instance to guess from." |
|---|
| 56 |
|
|---|
| 57 |
|
|---|
| 58 |
|
|---|
| 59 |
unbound = getattr(im_self.__class__,im_name) |
|---|
| 60 |
log.msg("Attempting fixup with",unbound) |
|---|
| 61 |
if im_self is None: |
|---|
| 62 |
return unbound |
|---|
| 63 |
bound=instancemethod(unbound.im_func, |
|---|
| 64 |
im_self, |
|---|
| 65 |
im_self.__class__) |
|---|
| 66 |
return bound |
|---|
| 67 |
|
|---|
| 68 |
copy_reg.pickle(types.MethodType, |
|---|
| 69 |
pickleMethod, |
|---|
| 70 |
unpickleMethod) |
|---|
| 71 |
|
|---|
| 72 |
def pickleModule(module): |
|---|
| 73 |
'support function for copy_reg to pickle module refs' |
|---|
| 74 |
return unpickleModule, (module.__name__,) |
|---|
| 75 |
|
|---|
| 76 |
def unpickleModule(name): |
|---|
| 77 |
'support function for copy_reg to unpickle module refs' |
|---|
| 78 |
if oldModules.has_key(name): |
|---|
| 79 |
log.msg("Module has moved: %s" % name) |
|---|
| 80 |
name = oldModules[name] |
|---|
| 81 |
log.msg(name) |
|---|
| 82 |
return __import__(name,{},{},'x') |
|---|
| 83 |
|
|---|
| 84 |
|
|---|
| 85 |
copy_reg.pickle(types.ModuleType, |
|---|
| 86 |
pickleModule, |
|---|
| 87 |
unpickleModule) |
|---|
| 88 |
|
|---|
| 89 |
def pickleStringO(stringo): |
|---|
| 90 |
'support function for copy_reg to pickle StringIO.OutputTypes' |
|---|
| 91 |
return unpickleStringO, (stringo.getvalue(), stringo.tell()) |
|---|
| 92 |
|
|---|
| 93 |
def unpickleStringO(val, sek): |
|---|
| 94 |
x = StringIO.StringIO() |
|---|
| 95 |
x.write(val) |
|---|
| 96 |
x.seek(sek) |
|---|
| 97 |
return x |
|---|
| 98 |
|
|---|
| 99 |
if hasattr(StringIO, 'OutputType'): |
|---|
| 100 |
copy_reg.pickle(StringIO.OutputType, |
|---|
| 101 |
pickleStringO, |
|---|
| 102 |
unpickleStringO) |
|---|
| 103 |
|
|---|
| 104 |
def pickleStringI(stringi): |
|---|
| 105 |
return unpickleStringI, (stringi.getvalue(), stringi.tell()) |
|---|
| 106 |
|
|---|
| 107 |
def unpickleStringI(val, sek): |
|---|
| 108 |
x = StringIO.StringIO(val) |
|---|
| 109 |
x.seek(sek) |
|---|
| 110 |
return x |
|---|
| 111 |
|
|---|
| 112 |
|
|---|
| 113 |
if hasattr(StringIO, 'InputType'): |
|---|
| 114 |
copy_reg.pickle(StringIO.InputType, |
|---|
| 115 |
pickleStringI, |
|---|
| 116 |
unpickleStringI) |
|---|
| 117 |
|
|---|
| 118 |
class Ephemeral: |
|---|
| 119 |
""" |
|---|
| 120 |
This type of object is never persisted; if possible, even references to it |
|---|
| 121 |
are eliminated. |
|---|
| 122 |
""" |
|---|
| 123 |
|
|---|
| 124 |
def __getstate__(self): |
|---|
| 125 |
log.msg( "WARNING: serializing ephemeral %s" % self ) |
|---|
| 126 |
import gc |
|---|
| 127 |
if getattr(gc, 'get_referrers', None): |
|---|
| 128 |
for r in gc.get_referrers(self): |
|---|
| 129 |
log.msg( " referred to by %s" % (r,)) |
|---|
| 130 |
return None |
|---|
| 131 |
|
|---|
| 132 |
def __setstate__(self, state): |
|---|
| 133 |
log.msg( "WARNING: unserializing ephemeral %s" % self.__class__ ) |
|---|
| 134 |
self.__class__ = Ephemeral |
|---|
| 135 |
|
|---|
| 136 |
|
|---|
| 137 |
versionedsToUpgrade = {} |
|---|
| 138 |
upgraded = {} |
|---|
| 139 |
|
|---|
| 140 |
def doUpgrade(): |
|---|
| 141 |
global versionedsToUpgrade, upgraded |
|---|
| 142 |
for versioned in versionedsToUpgrade.values(): |
|---|
| 143 |
requireUpgrade(versioned) |
|---|
| 144 |
versionedsToUpgrade = {} |
|---|
| 145 |
upgraded = {} |
|---|
| 146 |
|
|---|
| 147 |
def requireUpgrade(obj): |
|---|
| 148 |
"""Require that a Versioned instance be upgraded completely first. |
|---|
| 149 |
""" |
|---|
| 150 |
objID = id(obj) |
|---|
| 151 |
if objID in versionedsToUpgrade and objID not in upgraded: |
|---|
| 152 |
upgraded[objID] = 1 |
|---|
| 153 |
obj.versionUpgrade() |
|---|
| 154 |
return obj |
|---|
| 155 |
|
|---|
| 156 |
from twisted.python import reflect |
|---|
| 157 |
|
|---|
| 158 |
def _aybabtu(c): |
|---|
| 159 |
l = [] |
|---|
| 160 |
for b in reflect.allYourBase(c, Versioned): |
|---|
| 161 |
if b not in l and b is not Versioned: |
|---|
| 162 |
l.append(b) |
|---|
| 163 |
return l |
|---|
| 164 |
|
|---|
| 165 |
class Versioned: |
|---|
| 166 |
""" |
|---|
| 167 |
This type of object is persisted with versioning information. |
|---|
| 168 |
|
|---|
| 169 |
I have a single class attribute, the int persistenceVersion. After I am |
|---|
| 170 |
unserialized (and styles.doUpgrade() is called), self.upgradeToVersionX() |
|---|
| 171 |
will be called for each version upgrade I must undergo. |
|---|
| 172 |
|
|---|
| 173 |
For example, if I serialize an instance of a Foo(Versioned) at version 4 |
|---|
| 174 |
and then unserialize it when the code is at version 9, the calls:: |
|---|
| 175 |
|
|---|
| 176 |
self.upgradeToVersion5() |
|---|
| 177 |
self.upgradeToVersion6() |
|---|
| 178 |
self.upgradeToVersion7() |
|---|
| 179 |
self.upgradeToVersion8() |
|---|
| 180 |
self.upgradeToVersion9() |
|---|
| 181 |
|
|---|
| 182 |
will be made. If any of these methods are undefined, a warning message |
|---|
| 183 |
will be printed. |
|---|
| 184 |
""" |
|---|
| 185 |
persistenceVersion = 0 |
|---|
| 186 |
persistenceForgets = () |
|---|
| 187 |
|
|---|
| 188 |
def __setstate__(self, state): |
|---|
| 189 |
versionedsToUpgrade[id(self)] = self |
|---|
| 190 |
self.__dict__ = state |
|---|
| 191 |
|
|---|
| 192 |
def __getstate__(self, dict=None): |
|---|
| 193 |
"""Get state, adding a version number to it on its way out. |
|---|
| 194 |
""" |
|---|
| 195 |
dct = copy.copy(dict or self.__dict__) |
|---|
| 196 |
bases = _aybabtu(self.__class__) |
|---|
| 197 |
bases.reverse() |
|---|
| 198 |
bases.append(self.__class__) |
|---|
| 199 |
for base in bases: |
|---|
| 200 |
if base.__dict__.has_key('persistenceForgets'): |
|---|
| 201 |
for slot in base.persistenceForgets: |
|---|
| 202 |
if dct.has_key(slot): |
|---|
| 203 |
del dct[slot] |
|---|
| 204 |
if base.__dict__.has_key('persistenceVersion'): |
|---|
| 205 |
dct['%s.persistenceVersion' % reflect.qual(base)] = base.persistenceVersion |
|---|
| 206 |
return dct |
|---|
| 207 |
|
|---|
| 208 |
def versionUpgrade(self): |
|---|
| 209 |
"""(internal) Do a version upgrade. |
|---|
| 210 |
""" |
|---|
| 211 |
bases = _aybabtu(self.__class__) |
|---|
| 212 |
|
|---|
| 213 |
|
|---|
| 214 |
bases.reverse() |
|---|
| 215 |
bases.append(self.__class__) |
|---|
| 216 |
|
|---|
| 217 |
if self.__dict__.has_key("persistenceVersion"): |
|---|
| 218 |
|
|---|
| 219 |
|
|---|
| 220 |
|
|---|
| 221 |
|
|---|
| 222 |
|
|---|
| 223 |
|
|---|
| 224 |
|
|---|
| 225 |
|
|---|
| 226 |
|
|---|
| 227 |
pver = self.__dict__['persistenceVersion'] |
|---|
| 228 |
del self.__dict__['persistenceVersion'] |
|---|
| 229 |
highestVersion = 0 |
|---|
| 230 |
highestBase = None |
|---|
| 231 |
for base in bases: |
|---|
| 232 |
if not base.__dict__.has_key('persistenceVersion'): |
|---|
| 233 |
continue |
|---|
| 234 |
if base.persistenceVersion > highestVersion: |
|---|
| 235 |
highestBase = base |
|---|
| 236 |
highestVersion = base.persistenceVersion |
|---|
| 237 |
if highestBase: |
|---|
| 238 |
self.__dict__['%s.persistenceVersion' % reflect.qual(highestBase)] = pver |
|---|
| 239 |
for base in bases: |
|---|
| 240 |
|
|---|
| 241 |
if (Versioned not in base.__bases__ and |
|---|
| 242 |
not base.__dict__.has_key('persistenceVersion')): |
|---|
| 243 |
continue |
|---|
| 244 |
currentVers = base.persistenceVersion |
|---|
| 245 |
pverName = '%s.persistenceVersion' % reflect.qual(base) |
|---|
| 246 |
persistVers = (self.__dict__.get(pverName) or 0) |
|---|
| 247 |
if persistVers: |
|---|
| 248 |
del self.__dict__[pverName] |
|---|
| 249 |
assert persistVers <= currentVers, "Sorry, can't go backwards in time." |
|---|
| 250 |
while persistVers < currentVers: |
|---|
| 251 |
persistVers = persistVers + 1 |
|---|
| 252 |
method = base.__dict__.get('upgradeToVersion%s' % persistVers, None) |
|---|
| 253 |
if method: |
|---|
| 254 |
log.msg( "Upgrading %s (of %s @ %s) to version %s" % (reflect.qual(base), reflect.qual(self.__class__), id(self), persistVers) ) |
|---|
| 255 |
method(self) |
|---|
| 256 |
else: |
|---|
| 257 |
log.msg( 'Warning: cannot upgrade %s to version %s' % (base, persistVers) ) |
|---|