Ticket #5787: toptobottom-5.patch

File toptobottom-5.patch, 16.2 KB (added by Julian Berman, 9 years ago)
  • doc/core/man/trial.1

    diff --git a/doc/core/man/trial.1 b/doc/core/man/trial.1
    index 1bbdd58..00c491c 100644
    a b the default, except it makes tests run about ten times slower. 
    117117\fB-h\fR, \fB--help\fR
    118118Print a usage message to standard output, then exit.
    119119.TP
     120\fB--help-order\fR
     121Print a list of possible orders that TestCase test methods can be run in, then
     122exit. The orders can be used with the --order option described below.
     123.TP
    120124\fB--help-reporters\fR
    121125Print a list of valid reporters to standard output, then exit. Reporters can
    122126be selected with the --reporter option described below.
    every subpackage. Unless, that is, you specify this option. 
    141145Don't automatically jump into debugger for post-mortem analysis of
    142146exceptions.  Only usable in conjunction with --debug.
    143147.TP
     148\fB--order\fR \fIorder\fR
     149Specify what order to run the individual test methods within the given
     150TestCases. By default, they are run alphabetically. See --help-order for a list
     151of other valid values.
     152.TP
    144153\fB--profile\fR
    145154Run tests under the Python profiler.
    146155.TP
    Written by Jonathan M. Lange 
    204213.SH "REPORTING BUGS"
    205214To report a bug, visit http://twistedmatrix.com/trac/newticket
    206215.SH COPYRIGHT
    207 Copyright \(co 2003-2011 Twisted Matrix Laboratories
     216Copyright \(co 2003-2013 Twisted Matrix Laboratories
    208217.br
    209218This is free software; see the source for copying conditions.  There is NO
    210219warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  • twisted/scripts/trial.py

    diff --git a/twisted/scripts/trial.py b/twisted/scripts/trial.py
    index 46859d5..32a2835 100644
    a b  
    44# See LICENSE for details.
    55
    66
    7 import sys, os, random, gc, pdb, time, warnings
     7from __future__ import print_function
     8import dis
     9import gc
     10import inspect
     11import os
     12import pdb
     13import random
     14import sys
     15import time
     16import warnings
    817
    918from twisted.internet import defer
    1019from twisted.application import app
    def _reporterAction(): 
    94103
    95104
    96105
     106# orders which can be passed to trial --order
     107_runOrders = [
     108    ("alphabetical",
     109     "alphabetical order for test methods, arbitrary order for test cases"),
     110    ("toptobottom",
     111     "attempt to run test cases and methods in the order they were defined"),
     112]
     113
     114
     115
    97116class _BasicOptions(object):
    98117    """
    99118    Basic options shared between trial and its local workers.
    class _BasicOptions(object): 
    106125
    107126    optFlags = [["help", "h"],
    108127                ["no-recurse", "N", "Don't recurse into packages"],
     128                ['help-orders', None, "Help on available test running orders"],
    109129                ['help-reporters', None,
    110130                 "Help on available output plugins (reporters)"],
    111131                ["rterrors", "e", "realtime errors, print out tracebacks as "
    class _BasicOptions(object): 
    117137                ]
    118138
    119139    optParameters = [
     140        ["order", "o", None, "Specify what order to run test cases and methods"
     141         ". See --help-orders for more info."],
    120142        ["random", "z", None,
    121143         "Run tests in random order using the specified seed"],
    122144        ['temp-directory', None, '_trial_temp',
    class _BasicOptions(object): 
    126148         'more info.']]
    127149
    128150    compData = usage.Completions(
    129         optActions={"reporter": _reporterAction,
     151        optActions={"order": usage.CompleteList(
     152                        name for name, _ in _runOrders),
     153                    "reporter": _reporterAction,
    130154                    "logfile": usage.CompleteFiles(descr="log file name"),
    131155                    "random": usage.Completer(descr="random seed")},
    132156        extraActions=[usage.CompleteFiles(
    class _BasicOptions(object): 
    149173        """
    150174        coverdir = 'coverage'
    151175        result = FilePath(self['temp-directory']).child(coverdir)
    152         print "Setting coverage directory to %s." % (result.path,)
     176        print("Setting coverage directory to %s." % (result.path,))
    153177        return result
    154178
    155179
    class _BasicOptions(object): 
    196220        sys.settrace(spewer)
    197221
    198222
     223    def opt_help_orders(self):
     224        synopsis = ("Trial can attempt to run test cases and their methods in "
     225                    "a few different\n orders. You can select any of the "
     226                    "following options using --order=<foo>.\n")
     227
     228        print(synopsis)
     229        for name, description in _runOrders:
     230            print('   ', name, '\t', description)
     231        sys.exit(0)
     232
     233
    199234    def opt_help_reporters(self):
    200235        synopsis = ("Trial's output can be customized using plugins called "
    201236                    "Reporters. You can\nselect any of the following "
    202237                    "reporters using --reporter=<foo>\n")
    203         print synopsis
     238        print(synopsis)
    204239        for p in plugin.getPlugins(itrial.IReporter):
    205             print '   ', p.longOpt, '\t', p.description
    206         print
     240            print('   ', p.longOpt, '\t', p.description)
    207241        sys.exit(0)
    208242
    209243
    class _BasicOptions(object): 
    228262                "tbformat must be 'plain', 'emacs', or 'cgitb'.")
    229263
    230264
     265    def opt_order(self, order):
     266        """
     267        Run the tests in the given order.
     268
     269        @param order: a test ordering
     270        """
     271
     272        if order == "toptobottom":
     273            self['order'] = _maybeFindSourceLine
     274        elif order == "alphabetical":
     275            self['order'] = runner.name
     276        else:
     277            orders = ", ".join(repr(order) for order, _ in _runOrders)
     278            raise usage.UsageError("order must be one of " + orders)
     279
     280
    231281    def opt_recursionlimit(self, arg):
    232282        """
    233283        see sys.setrecursionlimit()
    def _getSuite(config): 
    403453
    404454
    405455
     456def _maybeFindSourceLine(thing):
     457    """
     458    Try to find the source line of the given test thing.
     459
     460
     461    @param testThing: a test method or class
     462    @rtype: int
     463    @return: the starting source line, or -1 if one couldn't be found
     464    """
     465
     466    method = getattr(thing, "_testMethodName", None)
     467    if method is not None:
     468        thing = getattr(thing, method)
     469
     470    # If it's a function, we can get the line number even if the source file no
     471    # longer exists
     472    code = getattr(thing, "func_code", None)
     473    if code is not None:
     474        _, startLine = next(dis.findlinestarts(code))
     475        return startLine
     476
     477    try:
     478        return inspect.getsourcelines(thing)[1]
     479    except (IOError, TypeError):
     480        # either thing is a module, which raised a TypeError, or the file
     481        # couldn't be read
     482        return -1
     483
     484
     485
    406486def _getLoader(config):
    407487    loader = runner.TestLoader()
    408488    if config['random']:
    409489        randomer = random.Random()
    410490        randomer.seed(config['random'])
    411491        loader.sorter = lambda x : randomer.random()
    412         print 'Running tests shuffled with seed %d\n' % config['random']
     492        print('Running tests shuffled with seed %d\n' % config['random'])
     493    elif config['order']:
     494        loader.sorter = config['order']
    413495    if not config['until-failure']:
    414496        loader.suiteFactory = runner.DestructiveTestSuite
    415497    return loader
    def _wrappedPdb(): 
    425507    try:
    426508        import readline
    427509    except ImportError:
    428         print "readline module not available"
     510        print("readline module not available")
    429511        sys.exc_clear()
    430512    for path in ('.pdbrc', 'pdbrc'):
    431513        if os.path.exists(path):
  • new file twisted/topfiles/5787.feature

    diff --git a/twisted/topfiles/5787.feature b/twisted/topfiles/5787.feature
    new file mode 100644
    index 0000000..5a98086
    - +  
     1trial now accepts an --order option that specifies what order to run TestCase methods in.
  • new file twisted/trial/test/ordertests.py

    diff --git a/twisted/trial/test/ordertests.py b/twisted/trial/test/ordertests.py
    new file mode 100644
    index 0000000..ee9c538
    - +  
     1# -*- test-case-name: twisted.trial.test.test_script -*-
     2# Copyright (c) Twisted Matrix Laboratories.
     3# See LICENSE for details.
     4
     5"""
     6Tests for handling of trial's --order option.
     7"""
     8
     9from twisted.trial import unittest
     10
     11
     12class FooTest(unittest.TestCase):
     13    """
     14    Used to make assertions about the order its tests will be run in.
     15    """
     16
     17    def test_first(self):
     18        pass
     19
     20
     21    def test_second(self):
     22        pass
     23
     24
     25    def test_third(self):
     26        pass
     27
     28
     29    def test_fourth(self):
     30        pass
     31
     32
     33
     34class BazTest(unittest.TestCase):
     35    """
     36    Used to make assertions about the order the test cases in this module are
     37    run in.
     38    """
     39    def test_baz(self):
     40        pass
     41
     42
     43
     44class BarTest(unittest.TestCase):
     45    """
     46    Used to make assertions about the order the test cases in this module are
     47    run in.
     48    """
     49    def test_bar(self):
     50        pass
  • twisted/trial/test/test_script.py

    diff --git a/twisted/trial/test/test_script.py b/twisted/trial/test/test_script.py
    index 45a254b..aa03550 100644
    a b  
    11# Copyright (c) Twisted Matrix Laboratories.
    22# See LICENSE for details.
    33
     4import StringIO
    45import gc
    5 import StringIO, sys, types
     6import re
     7import sys
     8import textwrap
     9import types
    610
    711from twisted.trial import unittest
    812from twisted.trial.runner import (
    class TestArgumentOrderTests(unittest.TestCase): 
    624628        expectedNames = testNames(expectedSuite)
    625629
    626630        self.assertEqual(names, expectedNames)
     631
     632
     633
     634class OrderTests(unittest.TestCase):
     635    """
     636    Tests for the --order option.
     637    """
     638    def setUp(self):
     639        self.config = trial.Options()
     640
     641
     642    def test_alphabetical(self):
     643        """
     644        --order=alphabetical causes trial to run tests alphabetically within
     645        each test case.
     646        """
     647        self.config.parseOptions([
     648            "--order", "alphabetical",
     649            "twisted.trial.test.ordertests.FooTest"])
     650
     651        loader = trial._getLoader(self.config)
     652        suite = loader.loadByNames(self.config['tests'])
     653
     654        self.assertEqual(
     655            testNames(suite), [
     656            'twisted.trial.test.ordertests.FooTest.test_first',
     657            'twisted.trial.test.ordertests.FooTest.test_fourth',
     658            'twisted.trial.test.ordertests.FooTest.test_second',
     659            'twisted.trial.test.ordertests.FooTest.test_third'])
     660
     661
     662    def test_alphabeticalModule(self):
     663        """
     664        --order=alphabetical causes trial to run test classes within a given
     665        module alphabetically.
     666        """
     667        self.config.parseOptions([
     668            "--order", "alphabetical", "twisted.trial.test.ordertests"])
     669        loader = trial._getLoader(self.config)
     670        suite = loader.loadByNames(self.config['tests'])
     671
     672        self.assertEqual(
     673            testNames(suite), [
     674            'twisted.trial.test.ordertests.BarTest.test_bar',
     675            'twisted.trial.test.ordertests.BazTest.test_baz',
     676            'twisted.trial.test.ordertests.FooTest.test_first',
     677            'twisted.trial.test.ordertests.FooTest.test_fourth',
     678            'twisted.trial.test.ordertests.FooTest.test_second',
     679            'twisted.trial.test.ordertests.FooTest.test_third'])
     680
     681
     682    def test_alphabeticalPackage(self):
     683        """
     684        --order=alphabetical causes trial to run test modules within a given
     685        package alphabetically, with tests within each module alphabetized.
     686        """
     687        self.config.parseOptions([
     688            "--order", "alphabetical", "twisted.trial.test"])
     689        loader = trial._getLoader(self.config)
     690        suite = loader.loadByNames(self.config['tests'])
     691
     692        names = testNames(suite)
     693        self.assertTrue(names, msg="Failed to load any tests!")
     694        self.assertEqual(names, sorted(names))
     695
     696
     697    def test_toptobottom(self):
     698        """
     699        --order=toptobottom causes trial to run test methods within a given
     700        test case from top to bottom as they are defined in the body of the
     701        class.
     702        """
     703        self.config.parseOptions([
     704            "--order", "toptobottom",
     705            "twisted.trial.test.ordertests.FooTest"])
     706
     707        loader = trial._getLoader(self.config)
     708        suite = loader.loadByNames(self.config['tests'])
     709
     710        self.assertEqual(
     711            testNames(suite), [
     712            'twisted.trial.test.ordertests.FooTest.test_first',
     713            'twisted.trial.test.ordertests.FooTest.test_second',
     714            'twisted.trial.test.ordertests.FooTest.test_third',
     715            'twisted.trial.test.ordertests.FooTest.test_fourth'])
     716
     717
     718    def test_toptobottomModule(self):
     719        """
     720        --order=toptobottom causes trial to run test classes within a given
     721        module from top to bottom as they are defined in the module's source.
     722        """
     723        self.config.parseOptions([
     724            "--order", "toptobottom", "twisted.trial.test.ordertests"])
     725        loader = trial._getLoader(self.config)
     726        suite = loader.loadByNames(self.config['tests'])
     727
     728        self.assertEqual(
     729            testNames(suite), [
     730            'twisted.trial.test.ordertests.FooTest.test_first',
     731            'twisted.trial.test.ordertests.FooTest.test_second',
     732            'twisted.trial.test.ordertests.FooTest.test_third',
     733            'twisted.trial.test.ordertests.FooTest.test_fourth',
     734            'twisted.trial.test.ordertests.BazTest.test_baz',
     735            'twisted.trial.test.ordertests.BarTest.test_bar'])
     736
     737
     738    def test_toptobottomPackage(self):
     739        """
     740        --order=toptobottom causes trial to run test modules within a given
     741        package alphabetically, with tests within each module run top to
     742        bottom.
     743        """
     744        self.config.parseOptions([
     745            "--order", "toptobottom", "twisted.trial.test"])
     746        loader = trial._getLoader(self.config)
     747        suite = loader.loadByNames(self.config['tests'])
     748
     749        names = testNames(suite)
     750        self.assertEqual(
     751            names, sorted(names, key=lambda name : name.partition(".")[0]),
     752        )
     753
     754
     755    def test_toptobottomMissingSource(self):
     756        """
     757        --order=toptobottom detects the source line of methods from modules
     758        whose source file is missing.
     759        """
     760        tempdir = self.mktemp().encode('utf-8')
     761        package = FilePath(tempdir).child(b'twisted_toptobottom_temp')
     762        package.makedirs()
     763        package.child(b'__init__.py').setContent(b'')
     764        package.child(b'test_missing.py').setContent(textwrap.dedent(b'''
     765        from twisted.trial.unittest import TestCase
     766        class TestMissing(TestCase):
     767            def test_second(self): pass
     768            def test_third(self): pass
     769            def test_fourth(self): pass
     770            def test_first(self): pass
     771        '''))
     772        pathEntry = package.parent().path.decode('utf-8')
     773        sys.path.insert(0, pathEntry)
     774        self.addCleanup(sys.path.remove, pathEntry)
     775        from twisted_toptobottom_temp import test_missing
     776        self.addCleanup(sys.modules.pop, 'twisted_toptobottom_temp')
     777        self.addCleanup(sys.modules.pop, test_missing.__name__)
     778        package.child(b'test_missing.py').remove()
     779
     780        self.config.parseOptions([
     781            "--order", "toptobottom", "twisted.trial.test.ordertests"])
     782        loader = trial._getLoader(self.config)
     783        suite = loader.loadModule(test_missing)
     784
     785        self.assertEqual(
     786            testNames(suite), [
     787            'twisted_toptobottom_temp.test_missing.TestMissing.test_second',
     788            'twisted_toptobottom_temp.test_missing.TestMissing.test_third',
     789            'twisted_toptobottom_temp.test_missing.TestMissing.test_fourth',
     790            'twisted_toptobottom_temp.test_missing.TestMissing.test_first'])
     791
     792
     793
     794class HelpOrderTests(unittest.TestCase):
     795    """
     796    Tests for the --help-orders flag.
     797    """
     798    def test_help_ordersPrintsSynopsisAndQuits(self):
     799        """
     800        --help-orders prints each of the available orders and then exits.
     801        """
     802        self.status = None
     803        self.patch(
     804            trial.sys, "exit", lambda status: setattr(self, "status", status))
     805        self.patch(sys, "stdout", StringIO.StringIO())
     806
     807        trial.Options().parseOptions(["--help-orders"])
     808
     809        for orderName, orderDesc in trial._runOrders:
     810            match = re.search(
     811                "{0}.*{1}".format(orderName, orderDesc), sys.stdout.getvalue(),
     812            )
     813            self.assertTrue(match)
     814        self.assertEqual(self.status, 0)