root / trunk / twisted / application / app.py

Revision 25666, 23.3 kB (checked in by exarkun, 7 months ago)

Merge delete-marmalade-876

Author: exarkun, itamarst
Reviewer: therve
Fixes: #876

Delete twisted.persisted.marmalade, a long-deprecated XML Python Object
serialization library, and all of the vestigal code and documentation in
Twisted which still referred to it.

Line 
1 # -*- test-case-name: twisted.test.test_application,twisted.test.test_twistd -*-
2 # Copyright (c) 2001-2008 Twisted Matrix Laboratories.
3 # See LICENSE for details.
4
5 import sys, os, pdb, getpass, traceback, signal, warnings
6
7 from twisted.python import runtime, log, usage, failure, util, logfile
8 from twisted.python.versions import Version
9 from twisted.python.reflect import qual
10 from twisted.python.deprecate import deprecated
11 from twisted.python.log import ILogObserver
12 from twisted.persisted import sob
13 from twisted.application import service, reactors
14 from twisted.internet import defer
15 from twisted import copyright
16
17 # Expose the new implementation of installReactor at the old location.
18 from twisted.application.reactors import installReactor
19 from twisted.application.reactors import NoSuchReactor
20
21
22
23 class _BasicProfiler(object):
24     """
25     @ivar saveStats: if C{True}, save the stats information instead of the
26         human readable format
27     @type saveStats: C{bool}
28
29     @ivar profileOutput: the name of the file use to print profile data.
30     @type profileOutput: C{str}
31     """
32
33     def __init__(self, profileOutput, saveStats):
34         self.profileOutput = profileOutput
35         self.saveStats = saveStats
36
37
38     def _reportImportError(self, module, e):
39         """
40         Helper method to report an import error with a profile module. This
41         has to be explicit because some of these modules are removed by
42         distributions due to them being non-free.
43         """
44         s = "Failed to import module %s: %s" % (module, e)
45         s += """
46 This is most likely caused by your operating system not including
47 the module due to it being non-free. Either do not use the option
48 --profile, or install the module; your operating system vendor
49 may provide it in a separate package.
50 """
51         raise SystemExit(s)
52
53
54
55 class ProfileRunner(_BasicProfiler):
56     """
57     Runner for the standard profile module.
58     """
59
60     def run(self, reactor):
61         """
62         Run reactor under the standard profiler.
63         """
64         try:
65             import profile
66         except ImportError, e:
67             self._reportImportError("profile", e)
68
69         p = profile.Profile()
70         p.runcall(reactor.run)
71         if self.saveStats:
72             p.dump_stats(self.profileOutput)
73         else:
74             tmp, sys.stdout = sys.stdout, open(self.profileOutput, 'a')
75             try:
76                 p.print_stats()
77             finally:
78                 sys.stdout, tmp = tmp, sys.stdout
79                 tmp.close()
80
81
82
83 class HotshotRunner(_BasicProfiler):
84     """
85     Runner for the hotshot profile module.
86     """
87
88     def run(self, reactor):
89         """
90         Run reactor under the hotshot profiler.
91         """
92         try:
93             import hotshot.stats
94         except (ImportError, SystemExit), e:
95             # Certain versions of Debian (and Debian derivatives) raise
96             # SystemExit when importing hotshot if the "non-free" profiler
97             # module is not installed.  Someone eventually recognized this
98             # as a bug and changed the Debian packaged Python to raise
99             # ImportError instead.  Handle both exception types here in
100             # order to support the versions of Debian which have this
101             # behavior.  The bug report which prompted the introduction of
102             # this highly undesirable behavior should be available online at
103             # <http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=334067>.
104             # There seems to be no corresponding bug report which resulted
105             # in the behavior being removed. -exarkun
106             self._reportImportError("hotshot", e)
107
108         # this writes stats straight out
109         p = hotshot.Profile(self.profileOutput)
110         p.runcall(reactor.run)
111         if self.saveStats:
112             # stats are automatically written to file, nothing to do
113             return
114         else:
115             s = hotshot.stats.load(self.profileOutput)
116             s.strip_dirs()
117             s.sort_stats(-1)
118             if getattr(s, 'stream', None) is not None:
119                 # Python 2.5 and above supports a stream attribute
120                 s.stream = open(self.profileOutput, 'w')
121                 s.print_stats()
122                 s.stream.close()
123             else:
124                 # But we have to use a trick for Python < 2.5
125                 tmp, sys.stdout = sys.stdout, open(self.profileOutput, 'w')
126                 try:
127                     s.print_stats()
128                 finally:
129                     sys.stdout, tmp = tmp, sys.stdout
130                     tmp.close()
131
132
133
134 class CProfileRunner(_BasicProfiler):
135     """
136     Runner for the cProfile module.
137     """
138
139     def run(self, reactor):
140         """
141         Run reactor under the cProfile profiler.
142         """
143         try:
144             import cProfile, pstats
145         except ImportError, e:
146             self._reportImportError("cProfile", e)
147
148         p = cProfile.Profile()
149         p.runcall(reactor.run)
150         if self.saveStats:
151             p.dump_stats(self.profileOutput)
152         else:
153             stream = open(self.profileOutput, 'w')
154             s = pstats.Stats(p, stream=stream)
155             s.strip_dirs()
156             s.sort_stats(-1)
157             s.print_stats()
158             stream.close()
159
160
161
162 class AppProfiler(object):
163     """
164     Class which selects a specific profile runner based on configuration
165     options.
166
167     @ivar profiler: the name of the selected profiler.
168     @type profiler: C{str}
169     """
170     profilers = {"profile": ProfileRunner, "hotshot": HotshotRunner,
171                  "cprofile": CProfileRunner}
172
173     def __init__(self, options):
174         saveStats = options.get("savestats", False)
175         profileOutput = options.get("profile", None)
176         self.profiler = options.get("profiler", "hotshot").lower()
177         if options.get("nothotshot", False):
178             warnings.warn("The --nothotshot option is deprecated. Please "
179                           "specify the profiler name using the --profiler "
180                           "option", category=DeprecationWarning)
181             self.profiler = "profile"
182         if self.profiler in self.profilers:
183             profiler = self.profilers[self.profiler](profileOutput, saveStats)
184             self.run = profiler.run
185         else:
186             raise SystemExit("Unsupported profiler name: %s" % (self.profiler,))
187
188
189
190 def runWithProfiler(reactor, config):
191     """
192     DEPRECATED in Twisted 8.0.
193
194     Run reactor under standard profiler.
195     """
196     warnings.warn("runWithProfiler is deprecated since Twisted 8.0. "
197                   "Use ProfileRunner instead.", DeprecationWarning, 2)
198     item = AppProfiler(config)
199     return item.run(reactor)
200
201
202
203 def runWithHotshot(reactor, config):
204     """
205     DEPRECATED in Twisted 8.0.
206
207     Run reactor under hotshot profiler.
208     """
209     warnings.warn("runWithHotshot is deprecated since Twisted 8.0. "
210                   "Use HotshotRunner instead.", DeprecationWarning, 2)
211     item = AppProfiler(config)
212     return item.run(reactor)
213
214
215
216 class AppLogger(object):
217     """
218     Class managing logging faciliy of the application.
219
220     @ivar _logfilename: The name of the file to which to log, if other than the
221         default.
222     @type _logfilename: C{str}
223
224     @ivar _observer: log observer added at C{start} and removed at C{stop}.
225     @type _observer: C{callable}
226     """
227     _observer = None
228
229     def __init__(self, options):
230         self._logfilename = options.get("logfile", "")
231
232
233     def start(self, application):
234         """
235         Initialize the logging system.
236
237         If an L{ILogObserver} component has been set on C{application}, then
238         it will be used as the log observer.  Otherwise a log observer will be
239         created based on the command-line options.
240         
241         @param application: The application on which to check for an
242             L{ILogObserver}.
243         """
244         observer = application.getComponent(ILogObserver, None)
245
246         if observer is None:
247             observer = self._getLogObserver()
248         self._observer = observer
249         log.startLoggingWithObserver(self._observer)
250         self._initialLog()
251
252
253     def _initialLog(self):
254         """
255         Print twistd start log message.
256         """
257         from twisted.internet import reactor
258         log.msg("twistd %s (%s %s) starting up." % (copyright.version,
259                                                    sys.executable,
260                                                    runtime.shortPythonVersion()))
261         log.msg('reactor class: %s.' % (qual(reactor.__class__),))
262
263
264     def _getLogObserver(self):
265         """
266         Create a log observer to be added to the logging system before running
267         this application.
268         """
269         if self._logfilename == '-' or not self._logfilename:
270             logFile = sys.stdout
271         else:
272             logFile = logfile.LogFile.fromFullPath(self._logfilename)
273         return log.FileLogObserver(logFile).emit
274
275
276     def stop(self):
277         """
278         Print twistd stop log message.
279         """
280         log.msg("Server Shut Down.")
281         if self._observer is not None:
282             log.removeObserver(self._observer)
283             self._observer = None
284
285
286
287 def fixPdb():
288     def do_stop(self, arg):
289         self.clear_all_breaks()
290         self.set_continue()
291         from twisted.internet import reactor
292         reactor.callLater(0, reactor.stop)
293         return 1
294
295     def help_stop(self):
296         print """stop - Continue execution, then cleanly shutdown the twisted reactor."""
297
298     def set_quit(self):
299         os._exit(0)
300
301     pdb.Pdb.set_quit = set_quit
302     pdb.Pdb.do_stop = do_stop
303     pdb.Pdb.help_stop = help_stop
304
305
306
307 def runReactorWithLogging(config, oldstdout, oldstderr, profiler=None, reactor=None):
308     """
309     Start the reactor, using profiling if specified by the configuration, and
310     log any error happening in the process.
311
312     @param config: configuration of the twistd application.
313     @type config: L{ServerOptions}
314
315     @param oldstdout: initial value of C{sys.stdout}.
316     @type oldstdout: C{file}
317
318     @param oldstderr: initial value of C{sys.stderr}.
319     @type oldstderr: C{file}
320
321     @param profiler: object used to run the reactor with profiling.
322     @type profiler: L{AppProfiler}
323
324     @param reactor: The reactor to use.  If C{None}, the global reactor will
325         be used.
326     """
327     if reactor is None:
328         from twisted.internet import reactor
329     try:
330         if config['profile']:
331             if profiler is not None:
332                 profiler.run(reactor)
333             else:
334                 # Backward compatible code
335                 if not config['nothotshot']:
336                     runWithHotshot(reactor, config)
337                 else:
338                     runWithProfiler(reactor, config)
339         elif config['debug']:
340             sys.stdout = oldstdout
341             sys.stderr = oldstderr
342             if runtime.platformType == 'posix':
343                 signal.signal(signal.SIGUSR2, lambda *args: pdb.set_trace())
344                 signal.signal(signal.SIGINT, lambda *args: pdb.set_trace())
345             fixPdb()
346             pdb.runcall(reactor.run)
347         else:
348             reactor.run()
349     except:
350         if config['nodaemon']:
351             file = oldstdout
352         else:
353             file = open("TWISTD-CRASH.log",'a')
354         traceback.print_exc(file=file)
355         file.flush()
356
357
358
359 def getPassphrase(needed):
360     if needed:
361         return getpass.getpass('Passphrase: ')
362     else:
363         return None
364
365
366
367 def getSavePassphrase(needed):
368     if needed:
369         passphrase = util.getPassword("Encryption passphrase: ")
370     else:
371         return None
372
373
374
375 class ApplicationRunner(object):
376     """
377     An object which helps running an application based on a config object.
378
379     Subclass me and implement preApplication and postApplication
380     methods. postApplication generally will want to run the reactor
381     after starting the application.
382
383     @ivar config: The config object, which provides a dict-like interface.
384
385     @ivar application: Available in postApplication, but not
386        preApplication. This is the application object.
387
388     @ivar profilerFactory: Factory for creating a profiler object, able to
389         profile the application if options are set accordingly.
390
391     @ivar profiler: Instance provided by C{profilerFactory}.
392
393     @ivar loggerFactory: Factory for creating object responsible for logging.
394
395     @ivar logger: Instance provided by C{loggerFactory}.
396     """
397     profilerFactory = AppProfiler
398     loggerFactory = AppLogger
399
400     def __init__(self, config):
401         self.config = config
402         self.profiler = self.profilerFactory(config)
403         self.logger = self.loggerFactory(config)
404
405
406     def run(self):
407         """
408         Run the application.
409         """
410         self.preApplication()
411         self.application = self.createOrGetApplication()
412
413
414         getLogObserverLegacy = getattr(self, 'getLogObserver', None)
415         if getLogObserverLegacy is not None:
416             warnings.warn("Specifying a log observer with getLogObserver is "
417                           "deprecated. Please use a loggerFactory instead.",
418                           category=DeprecationWarning)
419             self.startLogging(self.getLogObserver())
420         else:
421             self.logger.start(self.application)
422
423         self.postApplication()
424         self.logger.stop()
425
426
427     def startLogging(self, observer):
428         """
429         Initialize the logging system. DEPRECATED.
430
431         @param observer: The observer to add to the logging system.
432         """
433         log.startLoggingWithObserver(observer)
434         self.logger._initialLog()
435
436
437     def startReactor(self, reactor, oldstdout, oldstderr):
438         """
439         Run the reactor with the given configuration.  Subclasses should
440         probably call this from C{postApplication}.
441
442         @see: L{runReactorWithLogging}
443         """
444         runReactorWithLogging(
445             self.config, oldstdout, oldstderr, self.profiler, reactor)
446
447
448     def preApplication(self):
449         """
450         Override in subclass.
451
452         This should set up any state necessary before loading and
453         running the Application.
454         """
455         raise NotImplementedError()
456
457
458     def postApplication(self):
459         """
460         Override in subclass.
461
462         This will be called after the application has been loaded (so
463         the C{application} attribute will be set). Generally this
464         should start the application and run the reactor.
465         """
466         raise NotImplementedError()
467
468
469     def createOrGetApplication(self):
470         """
471         Create or load an Application based on the parameters found in the
472         given L{ServerOptions} instance.
473
474         If a subcommand was used, the L{service.IServiceMaker} that it
475         represents will be used to construct a service to be added to
476         a newly-created Application.
477
478         Otherwise, an application will be loaded based on parameters in
479         the config.
480         """
481         if self.config.subCommand:
482             # If a subcommand was given, it's our responsibility to create
483             # the application, instead of load it from a file.
484
485             # loadedPlugins is set up by the ServerOptions.subCommands
486             # property, which is iterated somewhere in the bowels of
487             # usage.Options.
488             plg = self.config.loadedPlugins[self.config.subCommand]
489             ser = plg.makeService(self.config.subOptions)
490             application = service.Application(plg.tapname)
491             ser.setServiceParent(application)
492         else:
493             passphrase = getPassphrase(self.config['encrypted'])
494             application = getApplication(self.config, passphrase)
495         return application
496
497
498
499 def getApplication(config, passphrase):
500     s = [(config[t], t)
501            for t in ['python', 'source', 'file'] if config[t]][0]
502     filename, style = s[0], {'file':'pickle'}.get(s[1],s[1])
503     try:
504         log.msg("Loading %s..." % filename)
505         application = service.loadApplication(filename, style, passphrase)
506         log.msg("Loaded.")
507     except Exception, e:
508         s = "Failed to load application: %s" % e
509         if isinstance(e, KeyError) and e.args[0] == "application":
510             s += """
511 Could not find 'application' in the file. To use 'twistd -y', your .tac
512 file must create a suitable object (e.g., by calling service.Application())
513 and store it in a variable named 'application'. twistd loads your .tac file
514 and scans the global variables for one of this name.
515
516 Please read the 'Using Application' HOWTO for details.
517 """
518         traceback.print_exc(file=log.logfile)
519         log.msg(s)
520         log.deferr()
521         sys.exit('\n' + s + '\n')
522     return application
523
524
525
526 def reportProfile(report_profile, name):
527     """
528     DEPRECATED since Twisted 8.0. This does nothing.
529     """
530     warnings.warn("reportProfile is deprecated and a no-op since Twisted 8.0.",
531                   category=DeprecationWarning)
532
533
534
535 def _reactorZshAction():
536     return "(%s)" % " ".join([r.shortName for r in reactors.getReactorTypes()])
537
538 class ReactorSelectionMixin:
539     """
540     Provides options for selecting a reactor to install.
541     """
542     zsh_actions = {"reactor" : _reactorZshAction}
543     messageOutput = sys.stdout
544
545
546     def opt_help_reactors(self):
547         """
548         Display a list of possibly available reactor names.
549         """
550         for r in reactors.getReactorTypes():
551             self.messageOutput.write('    %-4s\t%s\n' %
552                                      (r.shortName, r.description))
553         raise SystemExit(0)
554
555
556     def opt_reactor(self, shortName):
557         """
558         Which reactor to use (see --help-reactors for a list of possibilities)
559         """
560         # Actually actually actually install the reactor right at this very
561         # moment, before any other code (for example, a sub-command plugin)
562         # runs and accidentally imports and installs the default reactor.
563         #
564         # This could probably be improved somehow.
565         try:
566             installReactor(shortName)
567         except NoSuchReactor:
568             msg = ("The specified reactor does not exist: '%s'.\n"
569                    "See the list of available reactors with "
570                    "--help-reactors" % (shortName,))
571             raise usage.UsageError(msg)
572         except Exception, e:
573             msg = ("The specified reactor cannot be used, failed with error: "
574                    "%s.\nSee the list of available reactors with "
575                    "--help-reactors" % (e,))
576             raise usage.UsageError(msg)
577     opt_r = opt_reactor
578
579
580
581
582 class ServerOptions(usage.Options, ReactorSelectionMixin):
583
584     optFlags = [['savestats', None,
585                  "save the Stats object rather than the text output of "
586                  "the profiler."],
587                 ['no_save','o',   "do not save state on shutdown"],
588                 ['encrypted', 'e',
589                  "The specified tap/aos file is encrypted."],
590                 ['nothotshot', None,
591                  "DEPRECATED. Don't use the hotshot profiler even if "
592                  "it's available."]]
593
594     optParameters = [['logfile','l', None,
595                       "log to a specified file, - for stdout"],
596                      ['profile', 'p', None,
597                       "Run in profile mode, dumping results to specified file"],
598                      ['profiler', None, "hotshot",
599                       "Name of the profiler to use (%s)." %
600                       ", ".join(AppProfiler.profilers)],
601                      ['file','f','twistd.tap',
602                       "read the given .tap file"],
603                      ['python','y', None,
604                       "read an application from within a Python file "
605                       "(implies -o)"],
606                      ['source', 's', None,
607                       "Read an application from a .tas file (AOT format)."],
608                      ['rundir','d','.',
609                       'Change to a supplied directory before running'],
610                      ['report-profile', None, None,
611                       'E-mail address to use when reporting dynamic execution '
612                       'profiler stats.  This should not be combined with '
613                       'other profiling options.  This will only take effect '
614                       'if the application to be run has an application '
615                       'name.']]
616
617     #zsh_altArgDescr = {"foo":"use this description for foo instead"}
618     #zsh_multiUse = ["foo", "bar"]
619     zsh_mutuallyExclusive = [("file", "python", "source")]
620     zsh_actions = {"file":'_files -g "*.tap"',
621                    "python":'_files -g "*.(tac|py)"',
622                    "source":'_files -g "*.tas"',
623                    "rundir":"_dirs"}
624     #zsh_actionDescr = {"logfile":"log file name", "random":"random seed"}
625
626     def __init__(self, *a, **kw):
627         self['debug'] = False
628         usage.Options.__init__(self, *a, **kw)
629
630     def opt_debug(self):
631         """
632         run the application in the Python Debugger (implies nodaemon),
633         sending SIGUSR2 will drop into debugger
634         """
635         defer.setDebugging(True)
636         failure.startDebugMode()
637         self['debug'] = True
638     opt_b = opt_debug
639
640
641     def opt_spew(self):
642         """Print an insanely verbose log of everything that happens.
643         Useful when debugging freezes or locks in complex code."""
644         sys.settrace(util.spewer)
645         try:
646             import threading
647         except ImportError:
648             return
649         threading.settrace(util.spewer)
650
651
652     def opt_report_profile(self, value):
653         """
654         DEPRECATED.
655
656         Manage --report-profile option, which does nothing currently.
657         """
658         warnings.warn("--report-profile option is deprecated and a no-op "
659                       "since Twisted 8.0.", category=DeprecationWarning)
660
661
662     def parseOptions(self, options=None):
663         if options is None:
664             options = sys.argv[1:] or ["--help"]
665         usage.Options.parseOptions(self, options)
666
667     def postOptions(self):
668         if self.subCommand or self['python']:
669             self['no_save'] = True
670
671     def subCommands(self):
672         from twisted import plugin
673         plugins = plugin.getPlugins(service.IServiceMaker)
674         self.loadedPlugins = {}
675         for plug in plugins:
676             self.loadedPlugins[plug.tapname] = plug
677             yield (plug.tapname, None, lambda: plug.options(), plug.description)
678     subCommands = property(subCommands)
679
680
681
682 def run(runApp, ServerOptions):
683     config = ServerOptions()
684     try:
685         config.parseOptions()
686     except usage.error, ue:
687         print config
688         print "%s: %s" % (sys.argv[0], ue)
689     else:
690         runApp(config)
691
692
693
694 def initialLog():
695     AppLogger({})._initialLog()
696 initialLog = deprecated(Version("Twisted", 8, 2, 0))(initialLog)
697
698
699
700 def convertStyle(filein, typein, passphrase, fileout, typeout, encrypt):
701     application = service.loadApplication(filein, typein, passphrase)
702     sob.IPersistable(application).setStyle(typeout)
703     passphrase = getSavePassphrase(encrypt)
704     if passphrase:
705         fileout = None
706     sob.IPersistable(application).save(filename=fileout, passphrase=passphrase)
707
708 def startApplication(application, save):
709     from twisted.internet import reactor
710     service.IService(application).startService()
711     if save:
712          p = sob.IPersistable(application)
713          reactor.addSystemEventTrigger('after', 'shutdown', p.save, 'shutdown')
714     reactor.addSystemEventTrigger('before', 'shutdown',
715                                   service.IService(application).stopService)
716
717 def getLogFile(logfilename):
718     """
719     Build a log file from the full path.
720     """
721     warnings.warn(
722         "app.getLogFile is deprecated. Use "
723         "twisted.python.logfile.LogFile.fromFullPath instead",
724         DeprecationWarning, stacklevel=2)
725
726     return logfile.LogFile.fromFullPath(logfilename)
727
Note: See TracBrowser for help on using the browser.