root/trunk/twisted/application/app.py

Revision 33683, 21.6 KB (checked in by habnabit, 2 months ago)

Merge remove-initiallog-5480: Remove twisted.application.app.initialLog

Author: acdn01
Reviewer: habnabit
Fixes: #5480

twisted.application.app.initialLog was deprecated in Twisted 8.2.0 and is now
removed.

Line 
1# -*- test-case-name: twisted.test.test_application,twisted.test.test_twistd -*-
2# Copyright (c) Twisted Matrix Laboratories.
3# See LICENSE for details.
4
5import sys, os, pdb, getpass, traceback, signal
6from operator import attrgetter
7
8from twisted.python import runtime, log, usage, failure, util, logfile
9from twisted.python.versions import Version
10from twisted.python.reflect import qual, namedAny
11from twisted.python.deprecate import deprecated
12from twisted.python.log import ILogObserver
13from twisted.persisted import sob
14from twisted.application import service, reactors
15from twisted.internet import defer
16from twisted import copyright, plugin
17
18# Expose the new implementation of installReactor at the old location.
19from twisted.application.reactors import installReactor
20from twisted.application.reactors import NoSuchReactor
21
22
23
24class _BasicProfiler(object):
25    """
26    @ivar saveStats: if C{True}, save the stats information instead of the
27        human readable format
28    @type saveStats: C{bool}
29
30    @ivar profileOutput: the name of the file use to print profile data.
31    @type profileOutput: C{str}
32    """
33
34    def __init__(self, profileOutput, saveStats):
35        self.profileOutput = profileOutput
36        self.saveStats = saveStats
37
38
39    def _reportImportError(self, module, e):
40        """
41        Helper method to report an import error with a profile module. This
42        has to be explicit because some of these modules are removed by
43        distributions due to them being non-free.
44        """
45        s = "Failed to import module %s: %s" % (module, e)
46        s += """
47This is most likely caused by your operating system not including
48the module due to it being non-free. Either do not use the option
49--profile, or install the module; your operating system vendor
50may provide it in a separate package.
51"""
52        raise SystemExit(s)
53
54
55
56class ProfileRunner(_BasicProfiler):
57    """
58    Runner for the standard profile module.
59    """
60
61    def run(self, reactor):
62        """
63        Run reactor under the standard profiler.
64        """
65        try:
66            import profile
67        except ImportError, e:
68            self._reportImportError("profile", e)
69
70        p = profile.Profile()
71        p.runcall(reactor.run)
72        if self.saveStats:
73            p.dump_stats(self.profileOutput)
74        else:
75            tmp, sys.stdout = sys.stdout, open(self.profileOutput, 'a')
76            try:
77                p.print_stats()
78            finally:
79                sys.stdout, tmp = tmp, sys.stdout
80                tmp.close()
81
82
83
84class HotshotRunner(_BasicProfiler):
85    """
86    Runner for the hotshot profile module.
87    """
88
89    def run(self, reactor):
90        """
91        Run reactor under the hotshot profiler.
92        """
93        try:
94            import hotshot.stats
95        except (ImportError, SystemExit), e:
96            # Certain versions of Debian (and Debian derivatives) raise
97            # SystemExit when importing hotshot if the "non-free" profiler
98            # module is not installed.  Someone eventually recognized this
99            # as a bug and changed the Debian packaged Python to raise
100            # ImportError instead.  Handle both exception types here in
101            # order to support the versions of Debian which have this
102            # behavior.  The bug report which prompted the introduction of
103            # this highly undesirable behavior should be available online at
104            # <http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=334067>.
105            # There seems to be no corresponding bug report which resulted
106            # in the behavior being removed. -exarkun
107            self._reportImportError("hotshot", e)
108
109        # this writes stats straight out
110        p = hotshot.Profile(self.profileOutput)
111        p.runcall(reactor.run)
112        if self.saveStats:
113            # stats are automatically written to file, nothing to do
114            return
115        else:
116            s = hotshot.stats.load(self.profileOutput)
117            s.strip_dirs()
118            s.sort_stats(-1)
119            if getattr(s, 'stream', None) is not None:
120                # Python 2.5 and above supports a stream attribute
121                s.stream = open(self.profileOutput, 'w')
122                s.print_stats()
123                s.stream.close()
124            else:
125                # But we have to use a trick for Python < 2.5
126                tmp, sys.stdout = sys.stdout, open(self.profileOutput, 'w')
127                try:
128                    s.print_stats()
129                finally:
130                    sys.stdout, tmp = tmp, sys.stdout
131                    tmp.close()
132
133
134
135class CProfileRunner(_BasicProfiler):
136    """
137    Runner for the cProfile module.
138    """
139
140    def run(self, reactor):
141        """
142        Run reactor under the cProfile profiler.
143        """
144        try:
145            import cProfile, pstats
146        except ImportError, e:
147            self._reportImportError("cProfile", e)
148
149        p = cProfile.Profile()
150        p.runcall(reactor.run)
151        if self.saveStats:
152            p.dump_stats(self.profileOutput)
153        else:
154            stream = open(self.profileOutput, 'w')
155            s = pstats.Stats(p, stream=stream)
156            s.strip_dirs()
157            s.sort_stats(-1)
158            s.print_stats()
159            stream.close()
160
161
162
163class AppProfiler(object):
164    """
165    Class which selects a specific profile runner based on configuration
166    options.
167
168    @ivar profiler: the name of the selected profiler.
169    @type profiler: C{str}
170    """
171    profilers = {"profile": ProfileRunner, "hotshot": HotshotRunner,
172                 "cprofile": CProfileRunner}
173
174    def __init__(self, options):
175        saveStats = options.get("savestats", False)
176        profileOutput = options.get("profile", None)
177        self.profiler = options.get("profiler", "hotshot").lower()
178        if self.profiler in self.profilers:
179            profiler = self.profilers[self.profiler](profileOutput, saveStats)
180            self.run = profiler.run
181        else:
182            raise SystemExit("Unsupported profiler name: %s" % (self.profiler,))
183
184
185
186class AppLogger(object):
187    """
188    Class managing logging faciliy of the application.
189
190    @ivar _logfilename: The name of the file to which to log, if other than the
191        default.
192    @type _logfilename: C{str}
193
194    @ivar _observerFactory: Callable object that will create a log observer, or
195        None.
196
197    @ivar _observer: log observer added at C{start} and removed at C{stop}.
198    @type _observer: C{callable}
199    """
200    _observer = None
201
202    def __init__(self, options):
203        self._logfilename = options.get("logfile", "")
204        self._observerFactory = options.get("logger") or None
205
206
207    def start(self, application):
208        """
209        Initialize the logging system.
210
211        If a customer logger was specified on the command line it will be
212        used. If not, and an L{ILogObserver} component has been set on
213        C{application}, then it will be used as the log observer.  Otherwise a
214        log observer will be created based on the command-line options for
215        built-in loggers (e.g. C{--logfile}).
216
217        @param application: The application on which to check for an
218            L{ILogObserver}.
219        """
220        if self._observerFactory is not None:
221            observer = self._observerFactory()
222        else:
223            observer = application.getComponent(ILogObserver, None)
224
225        if observer is None:
226            observer = self._getLogObserver()
227        self._observer = observer
228        log.startLoggingWithObserver(self._observer)
229        self._initialLog()
230
231
232    def _initialLog(self):
233        """
234        Print twistd start log message.
235        """
236        from twisted.internet import reactor
237        log.msg("twistd %s (%s %s) starting up." % (copyright.version,
238                                                   sys.executable,
239                                                   runtime.shortPythonVersion()))
240        log.msg('reactor class: %s.' % (qual(reactor.__class__),))
241
242
243    def _getLogObserver(self):
244        """
245        Create a log observer to be added to the logging system before running
246        this application.
247        """
248        if self._logfilename == '-' or not self._logfilename:
249            logFile = sys.stdout
250        else:
251            logFile = logfile.LogFile.fromFullPath(self._logfilename)
252        return log.FileLogObserver(logFile).emit
253
254
255    def stop(self):
256        """
257        Print twistd stop log message.
258        """
259        log.msg("Server Shut Down.")
260        if self._observer is not None:
261            log.removeObserver(self._observer)
262            self._observer = None
263
264
265
266def fixPdb():
267    def do_stop(self, arg):
268        self.clear_all_breaks()
269        self.set_continue()
270        from twisted.internet import reactor
271        reactor.callLater(0, reactor.stop)
272        return 1
273
274
275    def help_stop(self):
276        print """stop - Continue execution, then cleanly shutdown the twisted reactor."""
277
278
279    def set_quit(self):
280        os._exit(0)
281
282    pdb.Pdb.set_quit = set_quit
283    pdb.Pdb.do_stop = do_stop
284    pdb.Pdb.help_stop = help_stop
285
286
287
288def runReactorWithLogging(config, oldstdout, oldstderr, profiler=None, reactor=None):
289    """
290    Start the reactor, using profiling if specified by the configuration, and
291    log any error happening in the process.
292
293    @param config: configuration of the twistd application.
294    @type config: L{ServerOptions}
295
296    @param oldstdout: initial value of C{sys.stdout}.
297    @type oldstdout: C{file}
298
299    @param oldstderr: initial value of C{sys.stderr}.
300    @type oldstderr: C{file}
301
302    @param profiler: object used to run the reactor with profiling.
303    @type profiler: L{AppProfiler}
304
305    @param reactor: The reactor to use.  If C{None}, the global reactor will
306        be used.
307    """
308    if reactor is None:
309        from twisted.internet import reactor
310    try:
311        if config['profile']:
312            if profiler is not None:
313                profiler.run(reactor)
314        elif config['debug']:
315            sys.stdout = oldstdout
316            sys.stderr = oldstderr
317            if runtime.platformType == 'posix':
318                signal.signal(signal.SIGUSR2, lambda *args: pdb.set_trace())
319                signal.signal(signal.SIGINT, lambda *args: pdb.set_trace())
320            fixPdb()
321            pdb.runcall(reactor.run)
322        else:
323            reactor.run()
324    except:
325        if config['nodaemon']:
326            file = oldstdout
327        else:
328            file = open("TWISTD-CRASH.log",'a')
329        traceback.print_exc(file=file)
330        file.flush()
331
332
333
334def getPassphrase(needed):
335    if needed:
336        return getpass.getpass('Passphrase: ')
337    else:
338        return None
339
340
341
342def getSavePassphrase(needed):
343    if needed:
344        passphrase = util.getPassword("Encryption passphrase: ")
345    else:
346        return None
347
348
349
350class ApplicationRunner(object):
351    """
352    An object which helps running an application based on a config object.
353
354    Subclass me and implement preApplication and postApplication
355    methods. postApplication generally will want to run the reactor
356    after starting the application.
357
358    @ivar config: The config object, which provides a dict-like interface.
359
360    @ivar application: Available in postApplication, but not
361       preApplication. This is the application object.
362
363    @ivar profilerFactory: Factory for creating a profiler object, able to
364        profile the application if options are set accordingly.
365
366    @ivar profiler: Instance provided by C{profilerFactory}.
367
368    @ivar loggerFactory: Factory for creating object responsible for logging.
369
370    @ivar logger: Instance provided by C{loggerFactory}.
371    """
372    profilerFactory = AppProfiler
373    loggerFactory = AppLogger
374
375    def __init__(self, config):
376        self.config = config
377        self.profiler = self.profilerFactory(config)
378        self.logger = self.loggerFactory(config)
379
380
381    def run(self):
382        """
383        Run the application.
384        """
385        self.preApplication()
386        self.application = self.createOrGetApplication()
387
388        self.logger.start(self.application)
389
390        self.postApplication()
391        self.logger.stop()
392
393
394    def startReactor(self, reactor, oldstdout, oldstderr):
395        """
396        Run the reactor with the given configuration.  Subclasses should
397        probably call this from C{postApplication}.
398
399        @see: L{runReactorWithLogging}
400        """
401        runReactorWithLogging(
402            self.config, oldstdout, oldstderr, self.profiler, reactor)
403
404
405    def preApplication(self):
406        """
407        Override in subclass.
408
409        This should set up any state necessary before loading and
410        running the Application.
411        """
412        raise NotImplementedError()
413
414
415    def postApplication(self):
416        """
417        Override in subclass.
418
419        This will be called after the application has been loaded (so
420        the C{application} attribute will be set). Generally this
421        should start the application and run the reactor.
422        """
423        raise NotImplementedError()
424
425
426    def createOrGetApplication(self):
427        """
428        Create or load an Application based on the parameters found in the
429        given L{ServerOptions} instance.
430
431        If a subcommand was used, the L{service.IServiceMaker} that it
432        represents will be used to construct a service to be added to
433        a newly-created Application.
434
435        Otherwise, an application will be loaded based on parameters in
436        the config.
437        """
438        if self.config.subCommand:
439            # If a subcommand was given, it's our responsibility to create
440            # the application, instead of load it from a file.
441
442            # loadedPlugins is set up by the ServerOptions.subCommands
443            # property, which is iterated somewhere in the bowels of
444            # usage.Options.
445            plg = self.config.loadedPlugins[self.config.subCommand]
446            ser = plg.makeService(self.config.subOptions)
447            application = service.Application(plg.tapname)
448            ser.setServiceParent(application)
449        else:
450            passphrase = getPassphrase(self.config['encrypted'])
451            application = getApplication(self.config, passphrase)
452        return application
453
454
455
456def getApplication(config, passphrase):
457    s = [(config[t], t)
458           for t in ['python', 'source', 'file'] if config[t]][0]
459    filename, style = s[0], {'file':'pickle'}.get(s[1],s[1])
460    try:
461        log.msg("Loading %s..." % filename)
462        application = service.loadApplication(filename, style, passphrase)
463        log.msg("Loaded.")
464    except Exception, e:
465        s = "Failed to load application: %s" % e
466        if isinstance(e, KeyError) and e.args[0] == "application":
467            s += """
468Could not find 'application' in the file. To use 'twistd -y', your .tac
469file must create a suitable object (e.g., by calling service.Application())
470and store it in a variable named 'application'. twistd loads your .tac file
471and scans the global variables for one of this name.
472
473Please read the 'Using Application' HOWTO for details.
474"""
475        traceback.print_exc(file=log.logfile)
476        log.msg(s)
477        log.deferr()
478        sys.exit('\n' + s + '\n')
479    return application
480
481
482
483def _reactorAction():
484    return usage.CompleteList([r.shortName for r in reactors.getReactorTypes()])
485
486
487class ReactorSelectionMixin:
488    """
489    Provides options for selecting a reactor to install.
490
491    If a reactor is installed, the short name which was used to locate it is
492    saved as the value for the C{"reactor"} key.
493    """
494    compData = usage.Completions(
495        optActions={"reactor": _reactorAction})
496
497    messageOutput = sys.stdout
498    _getReactorTypes = staticmethod(reactors.getReactorTypes)
499
500
501    def opt_help_reactors(self):
502        """
503        Display a list of possibly available reactor names.
504        """
505        rcts = sorted(self._getReactorTypes(), key=attrgetter('shortName'))
506        for r in rcts:
507            self.messageOutput.write('    %-4s\t%s\n' %
508                                     (r.shortName, r.description))
509        raise SystemExit(0)
510
511
512    def opt_reactor(self, shortName):
513        """
514        Which reactor to use (see --help-reactors for a list of possibilities)
515        """
516        # Actually actually actually install the reactor right at this very
517        # moment, before any other code (for example, a sub-command plugin)
518        # runs and accidentally imports and installs the default reactor.
519        #
520        # This could probably be improved somehow.
521        try:
522            installReactor(shortName)
523        except NoSuchReactor:
524            msg = ("The specified reactor does not exist: '%s'.\n"
525                   "See the list of available reactors with "
526                   "--help-reactors" % (shortName,))
527            raise usage.UsageError(msg)
528        except Exception, e:
529            msg = ("The specified reactor cannot be used, failed with error: "
530                   "%s.\nSee the list of available reactors with "
531                   "--help-reactors" % (e,))
532            raise usage.UsageError(msg)
533        else:
534            self["reactor"] = shortName
535    opt_r = opt_reactor
536
537
538
539
540class ServerOptions(usage.Options, ReactorSelectionMixin):
541
542    longdesc = ("twistd reads a twisted.application.service.Application out "
543                "of a file and runs it.")
544
545    optFlags = [['savestats', None,
546                 "save the Stats object rather than the text output of "
547                 "the profiler."],
548                ['no_save','o',   "do not save state on shutdown"],
549                ['encrypted', 'e',
550                 "The specified tap/aos file is encrypted."]]
551
552    optParameters = [['logfile','l', None,
553                      "log to a specified file, - for stdout"],
554                     ['logger', None, None,
555                      "A fully-qualified name to a log observer factory to use "
556                      "for the initial log observer.  Takes precedence over "
557                      "--logfile and --syslog (when available)."],
558                     ['profile', 'p', None,
559                      "Run in profile mode, dumping results to specified file"],
560                     ['profiler', None, "hotshot",
561                      "Name of the profiler to use (%s)." %
562                      ", ".join(AppProfiler.profilers)],
563                     ['file','f','twistd.tap',
564                      "read the given .tap file"],
565                     ['python','y', None,
566                      "read an application from within a Python file "
567                      "(implies -o)"],
568                     ['source', 's', None,
569                      "Read an application from a .tas file (AOT format)."],
570                     ['rundir','d','.',
571                      'Change to a supplied directory before running']]
572
573    compData = usage.Completions(
574        mutuallyExclusive=[("file", "python", "source")],
575        optActions={"file": usage.CompleteFiles("*.tap"),
576                    "python": usage.CompleteFiles("*.(tac|py)"),
577                    "source": usage.CompleteFiles("*.tas"),
578                    "rundir": usage.CompleteDirs()}
579        )
580
581    _getPlugins = staticmethod(plugin.getPlugins)
582
583    def __init__(self, *a, **kw):
584        self['debug'] = False
585        usage.Options.__init__(self, *a, **kw)
586
587
588    def opt_debug(self):
589        """
590        Run the application in the Python Debugger (implies nodaemon),
591        sending SIGUSR2 will drop into debugger
592        """
593        defer.setDebugging(True)
594        failure.startDebugMode()
595        self['debug'] = True
596    opt_b = opt_debug
597
598
599    def opt_spew(self):
600        """
601        Print an insanely verbose log of everything that happens.
602        Useful when debugging freezes or locks in complex code."""
603        sys.settrace(util.spewer)
604        try:
605            import threading
606        except ImportError:
607            return
608        threading.settrace(util.spewer)
609
610
611    def parseOptions(self, options=None):
612        if options is None:
613            options = sys.argv[1:] or ["--help"]
614        usage.Options.parseOptions(self, options)
615
616
617    def postOptions(self):
618        if self.subCommand or self['python']:
619            self['no_save'] = True
620        if self['logger'] is not None:
621            try:
622                self['logger'] = namedAny(self['logger'])
623            except Exception, e:
624                raise usage.UsageError("Logger '%s' could not be imported: %s" 
625                                       % (self['logger'], e))
626
627
628    def subCommands(self):
629        plugins = self._getPlugins(service.IServiceMaker)
630        self.loadedPlugins = {}
631        for plug in sorted(plugins, key=attrgetter('tapname')):
632            self.loadedPlugins[plug.tapname] = plug
633            yield (plug.tapname,
634                   None,
635                   # Avoid resolving the options attribute right away, in case
636                   # it's a property with a non-trivial getter (eg, one which
637                   # imports modules).
638                   lambda plug=plug: plug.options(),
639                   plug.description)
640    subCommands = property(subCommands)
641
642
643
644def run(runApp, ServerOptions):
645    config = ServerOptions()
646    try:
647        config.parseOptions()
648    except usage.error, ue:
649        print config
650        print "%s: %s" % (sys.argv[0], ue)
651    else:
652        runApp(config)
653
654
655
656def convertStyle(filein, typein, passphrase, fileout, typeout, encrypt):
657    application = service.loadApplication(filein, typein, passphrase)
658    sob.IPersistable(application).setStyle(typeout)
659    passphrase = getSavePassphrase(encrypt)
660    if passphrase:
661        fileout = None
662    sob.IPersistable(application).save(filename=fileout, passphrase=passphrase)
663
664
665
666def startApplication(application, save):
667    from twisted.internet import reactor
668    service.IService(application).startService()
669    if save:
670         p = sob.IPersistable(application)
671         reactor.addSystemEventTrigger('after', 'shutdown', p.save, 'shutdown')
672    reactor.addSystemEventTrigger('before', 'shutdown',
673                                  service.IService(application).stopService)
Note: See TracBrowser for help on using the browser.