Ticket #5787: toptobottom-4.patch

File toptobottom-4.patch, 16.0 KB (added by Julian Berman, 9 years ago)
  • doc/core/man/trial.1

    diff --git a/doc/core/man/trial.1 b/doc/core/man/trial.1
    index 1bbdd58..e1a8f05 100644
    a b the default, except it makes tests run about ten times slower. 
    117117\fB-h\fR, \fB--help\fR
    118118Print a usage message to standard output, then exit.
    119119.TP
     120\fB--help-order\fR
     121Print a list of possible orders that TestCase test methods can be run in, then
     122exit. The orders can be used with the --order option described below.
     123.TP
    120124\fB--help-reporters\fR
    121125Print a list of valid reporters to standard output, then exit. Reporters can
    122126be selected with the --reporter option described below.
    every subpackage. Unless, that is, you specify this option. 
    141145Don't automatically jump into debugger for post-mortem analysis of
    142146exceptions.  Only usable in conjunction with --debug.
    143147.TP
     148\fB--order\fR \fIorder\fR
     149Specify what order to run the individual test methods within the given
     150TestCases. By default, they are run alphabetically. See --help-order for a list
     151of other valid values.
     152.TP
    144153\fB--profile\fR
    145154Run tests under the Python profiler.
    146155.TP
  • twisted/scripts/trial.py

    diff --git a/twisted/scripts/trial.py b/twisted/scripts/trial.py
    index 307d67c..a7cfde6 100644
    a b  
    44# See LICENSE for details.
    55
    66
    7 import sys, os, random, gc, pdb, time, warnings
     7from __future__ import print_function
     8import sys, os, random, gc, pdb, time, warnings, inspect, dis
    89
    910from twisted.internet import defer
    1011from twisted.application import app
    def _reporterAction(): 
    9596
    9697
    9798
     99# orders which can be passed to trial --order
     100_runOrders = [
     101    ("alphabetical",
     102     "alphabetical order for test methods, arbitrary order for test cases"),
     103    ("toptobottom",
     104     "attempt to run test cases and methods in the order they were defined"),
     105]
     106
     107
     108
    98109class _BasicOptions(object):
    99110    """
    100111    Basic options shared between trial and its local workers.
    class _BasicOptions(object): 
    107118
    108119    optFlags = [["help", "h"],
    109120                ["no-recurse", "N", "Don't recurse into packages"],
     121                ['help-orders', None, "Help on available test running orders"],
    110122                ['help-reporters', None,
    111123                 "Help on available output plugins (reporters)"],
    112124                ["rterrors", "e", "realtime errors, print out tracebacks as "
    class _BasicOptions(object): 
    118130                ]
    119131
    120132    optParameters = [
     133        ["order", "o", None, "Specify what order to run test cases and methods"
     134         ". See --help-orders for more info."],
    121135        ["random", "z", None,
    122136         "Run tests in random order using the specified seed"],
    123137        ['temp-directory', None, '_trial_temp',
    class _BasicOptions(object): 
    127141         'more info.']]
    128142
    129143    compData = usage.Completions(
    130         optActions={"reporter": _reporterAction,
     144        optActions={"order": usage.CompleteList(
     145                        name for name, _ in _runOrders),
     146                    "reporter": _reporterAction,
    131147                    "logfile": usage.CompleteFiles(descr="log file name"),
    132148                    "random": usage.Completer(descr="random seed")},
    133149        extraActions=[usage.CompleteFiles(
    class _BasicOptions(object): 
    150166        """
    151167        coverdir = 'coverage'
    152168        result = FilePath(self['temp-directory']).child(coverdir)
    153         print "Setting coverage directory to %s." % (result.path,)
     169        print("Setting coverage directory to %s." % (result.path,))
    154170        return result
    155171
    156172
    class _BasicOptions(object): 
    197213        sys.settrace(spewer)
    198214
    199215
     216    def opt_help_orders(self):
     217        synopsis = ("Trial can attempt to run test cases and their methods in "
     218                    "a few different\n orders. You can select any of the "
     219                    "following options using --order=<foo>.\n")
     220
     221        print(synopsis)
     222        for name, description in _runOrders:
     223            print('   ', name, '\t', description)
     224        sys.exit(0)
     225
     226
    200227    def opt_help_reporters(self):
    201228        synopsis = ("Trial's output can be customized using plugins called "
    202229                    "Reporters. You can\nselect any of the following "
    203230                    "reporters using --reporter=<foo>\n")
    204         print synopsis
     231        print(synopsis)
    205232        for p in plugin.getPlugins(itrial.IReporter):
    206             print '   ', p.longOpt, '\t', p.description
    207         print
     233            print('   ', p.longOpt, '\t', p.description)
    208234        sys.exit(0)
    209235
    210236
    class _BasicOptions(object): 
    229255                "tbformat must be 'plain', 'emacs', or 'cgitb'.")
    230256
    231257
     258    def opt_order(self, order):
     259        """
     260        Run the tests in the given order.
     261
     262        """
     263
     264        if order == "toptobottom":
     265            self['order'] = _maybeFindSourceLine
     266        elif order == "alphabetical":
     267            self['order'] = runner.name
     268        else:
     269            orders = ", ".join(repr(order) for order, _ in _runOrders)
     270            raise usage.UsageError("order must be one of " + orders)
     271
     272
    232273    def opt_recursionlimit(self, arg):
    233274        """
    234275        see sys.setrecursionlimit()
    def _getSuite(config): 
    404445
    405446
    406447
     448def _maybeFindSourceLine(thing):
     449    """
     450    Try to find the source line of the given test thing.
     451
     452    """
     453
     454    method = getattr(thing, "_testMethodName", None)
     455    if method is not None:
     456        thing = getattr(thing, method)
     457
     458    # If it's a function, we can get the line number even if the source file no
     459    # longer exists
     460    code = getattr(thing, "func_code", None)
     461    if code is not None:
     462        _, startLine = next(dis.findlinestarts(code))
     463        return startLine
     464
     465    try:
     466        return inspect.getsourcelines(thing)[1]
     467    except (IOError, TypeError):
     468        # either thing is a module, which raised a TypeError, or the file
     469        # couldn't be read
     470        return -1
     471
     472
     473
    407474def _getLoader(config):
    408475    loader = runner.TestLoader()
    409476    if config['random']:
    410477        randomer = random.Random()
    411478        randomer.seed(config['random'])
    412479        loader.sorter = lambda x : randomer.random()
    413         print 'Running tests shuffled with seed %d\n' % config['random']
     480        print('Running tests shuffled with seed %d\n' % config['random'])
     481    elif config['order']:
     482        loader.sorter = config['order']
    414483    if not config['until-failure']:
    415484        loader.suiteFactory = runner.DestructiveTestSuite
    416485    return loader
    def _wrappedPdb(): 
    426495    try:
    427496        import readline
    428497    except ImportError:
    429         print "readline module not available"
     498        print("readline module not available")
    430499        sys.exc_clear()
    431500    for path in ('.pdbrc', 'pdbrc'):
    432501        if os.path.exists(path):
  • new file twisted/topfiles/5787.feature

    diff --git a/twisted/topfiles/5787.feature b/twisted/topfiles/5787.feature
    new file mode 100644
    index 0000000..7a7d97b
    - +  
     1trial now accepts a --order option that specifies what order to run TestCase methods in.
  • new file twisted/trial/test/ordertests.py

    diff --git a/twisted/trial/test/ordertests.py b/twisted/trial/test/ordertests.py
    new file mode 100644
    index 0000000..86f3c35
    - +  
     1# -*- test-case-name: twisted.trial.test.test_script -*-
     2# Copyright (c) Twisted Matrix Laboratories.
     3# See LICENSE for details.
     4
     5"""
     6Tests for handling of trial's --order option.
     7"""
     8
     9from twisted.trial import unittest
     10
     11
     12class FooTest(unittest.TestCase):
     13    """
     14    Used to make assertions about the order its tests will be run in.
     15    """
     16
     17    def test_first(self):
     18        pass
     19
     20
     21    def test_second(self):
     22        pass
     23
     24
     25    def test_third(self):
     26        pass
     27
     28
     29    def test_fourth(self):
     30        pass
     31
     32
     33
     34class BazTest(unittest.TestCase):
     35    def test_baz(self):
     36        pass
     37
     38
     39
     40class BarTest(unittest.TestCase):
     41    def test_bar(self):
     42        pass
     43# -*- test-case-name: twisted.trial.test.test_script -*-
     44# Copyright (c) Twisted Matrix Laboratories.
     45# See LICENSE for details.
     46
     47"""
     48Tests for handling of trial's --order option.
     49"""
     50
     51from twisted.trial import unittest
     52
     53
     54class FooTest(unittest.TestCase):
     55    """
     56    Used to make assertions about the order its tests will be run in.
     57    """
     58
     59    def test_first(self):
     60        pass
     61
     62
     63    def test_second(self):
     64        pass
     65
     66
     67    def test_third(self):
     68        pass
     69
     70
     71    def test_fourth(self):
     72        pass
     73
     74
     75
     76class BazTest(unittest.TestCase):
     77    def test_baz(self):
     78        pass
     79
     80
     81
     82class BarTest(unittest.TestCase):
     83    def test_bar(self):
     84        pass
  • twisted/trial/test/test_script.py

    diff --git a/twisted/trial/test/test_script.py b/twisted/trial/test/test_script.py
    index e71fb1a..178e2a1 100644
    a b  
    22# See LICENSE for details.
    33
    44import gc
     5import re
    56import StringIO, sys, types
     7import textwrap
    68
    79from twisted.trial import unittest
    810from twisted.trial.runner import (
    class TestArgumentOrderTests(unittest.TestCase): 
    625627        expectedNames = testNames(expectedSuite)
    626628
    627629        self.assertEqual(names, expectedNames)
     630
     631
     632
     633class OrderTests(unittest.TestCase):
     634    """
     635    Tests for the --order option.
     636    """
     637
     638
     639    def setUp(self):
     640        self.config = trial.Options()
     641
     642
     643    def test_alphabetical(self):
     644        """
     645        --order=alphabetical causes trial to run tests alphabetically within
     646        each test case.
     647        """
     648
     649        self.config.parseOptions([
     650            "--order", "alphabetical",
     651            "twisted.trial.test.ordertests.FooTest"])
     652
     653        loader = trial._getLoader(self.config)
     654        suite = loader.loadByNames(self.config['tests'])
     655
     656        self.assertEqual(
     657            testNames(suite), [
     658            'twisted.trial.test.ordertests.FooTest.test_first',
     659            'twisted.trial.test.ordertests.FooTest.test_fourth',
     660            'twisted.trial.test.ordertests.FooTest.test_second',
     661            'twisted.trial.test.ordertests.FooTest.test_third'])
     662
     663
     664    def test_alphabeticalModule(self):
     665        """
     666        --order=alphabetical causes trial to run test classes within a given
     667        module alphabetically.
     668        """
     669
     670        self.config.parseOptions([
     671            "--order", "alphabetical", "twisted.trial.test.ordertests"])
     672        loader = trial._getLoader(self.config)
     673        suite = loader.loadByNames(self.config['tests'])
     674
     675        self.assertEqual(
     676            testNames(suite), [
     677            'twisted.trial.test.ordertests.BarTest.test_bar',
     678            'twisted.trial.test.ordertests.BazTest.test_baz',
     679            'twisted.trial.test.ordertests.FooTest.test_first',
     680            'twisted.trial.test.ordertests.FooTest.test_fourth',
     681            'twisted.trial.test.ordertests.FooTest.test_second',
     682            'twisted.trial.test.ordertests.FooTest.test_third'])
     683
     684
     685    def test_alphabeticalPackage(self):
     686        """
     687        --order=alphabetical causes trial to run test modules within a given
     688        package alphabetically, with tests within each module alphabetized.
     689        """
     690
     691        self.config.parseOptions([
     692            "--order", "alphabetical", "twisted.trial.test"])
     693        loader = trial._getLoader(self.config)
     694        suite = loader.loadByNames(self.config['tests'])
     695
     696        names = testNames(suite)
     697        self.assertTrue(names, msg="Failed to load any tests!")
     698        self.assertEqual(names, sorted(names))
     699
     700
     701    def test_toptobottom(self):
     702        """
     703        --order=toptobottom causes trial to run test methods within a given
     704        test case from top to bottom as they are defined in the body of the
     705        class.
     706        """
     707
     708        self.config.parseOptions([
     709            "--order", "toptobottom",
     710            "twisted.trial.test.ordertests.FooTest"])
     711
     712        loader = trial._getLoader(self.config)
     713        suite = loader.loadByNames(self.config['tests'])
     714
     715        self.assertEqual(
     716            testNames(suite), [
     717            'twisted.trial.test.ordertests.FooTest.test_first',
     718            'twisted.trial.test.ordertests.FooTest.test_second',
     719            'twisted.trial.test.ordertests.FooTest.test_third',
     720            'twisted.trial.test.ordertests.FooTest.test_fourth'])
     721
     722
     723    def test_toptobottomModule(self):
     724        """
     725        --order=toptobottom causes trial to run test classes within a given
     726        module from top to bottom as they are defined in the module's source.
     727        """
     728
     729        self.config.parseOptions([
     730            "--order", "toptobottom", "twisted.trial.test.ordertests"])
     731        loader = trial._getLoader(self.config)
     732        suite = loader.loadByNames(self.config['tests'])
     733
     734        self.assertEqual(
     735            testNames(suite), [
     736            'twisted.trial.test.ordertests.FooTest.test_first',
     737            'twisted.trial.test.ordertests.FooTest.test_second',
     738            'twisted.trial.test.ordertests.FooTest.test_third',
     739            'twisted.trial.test.ordertests.FooTest.test_fourth',
     740            'twisted.trial.test.ordertests.BazTest.test_baz',
     741            'twisted.trial.test.ordertests.BarTest.test_bar'])
     742
     743
     744    def test_toptobottomPackage(self):
     745        """
     746        --order=toptobottom causes trial to run test modules within a given
     747        package alphabetically, with tests within each module run top to
     748        bottom.
     749        """
     750
     751        self.config.parseOptions([
     752            "--order", "toptobottom", "twisted.trial.test"])
     753        loader = trial._getLoader(self.config)
     754        suite = loader.loadByNames(self.config['tests'])
     755
     756        names = testNames(suite)
     757        self.assertEqual(
     758            names, sorted(names, key=lambda name : name.partition(".")[0]),
     759        )
     760
     761
     762    def test_toptobottomMissingSource(self):
     763        """
     764        --order=toptobottom detects the source line of methods from modules
     765        whose source file is missing.
     766        """
     767
     768        tempdir = self.mktemp().encode('utf-8')
     769        package = FilePath(tempdir).child(b'twisted_toptobottom_temp')
     770        package.makedirs()
     771        package.child(b'__init__.py').setContent(b'')
     772        package.child(b'test_missing.py').setContent(textwrap.dedent(b'''
     773        from twisted.trial.unittest import TestCase
     774        class TestMissing(TestCase):
     775            def test_second(self): pass
     776            def test_third(self): pass
     777            def test_fourth(self): pass
     778            def test_first(self): pass
     779        '''))
     780        pathEntry = package.parent().path.decode('utf-8')
     781        sys.path.insert(0, pathEntry)
     782        self.addCleanup(sys.path.remove, pathEntry)
     783        from twisted_toptobottom_temp import test_missing
     784        self.addCleanup(sys.modules.pop, 'twisted_toptobottom_temp')
     785        self.addCleanup(sys.modules.pop, test_missing.__name__)
     786        package.child(b'test_missing.py').remove()
     787
     788        self.config.parseOptions([
     789            "--order", "toptobottom", "twisted.trial.test.ordertests"])
     790        loader = trial._getLoader(self.config)
     791        suite = loader.loadModule(test_missing)
     792
     793        self.assertEqual(
     794            testNames(suite), [
     795            'twisted_toptobottom_temp.test_missing.TestMissing.test_second',
     796            'twisted_toptobottom_temp.test_missing.TestMissing.test_third',
     797            'twisted_toptobottom_temp.test_missing.TestMissing.test_fourth',
     798            'twisted_toptobottom_temp.test_missing.TestMissing.test_first'])
     799
     800
     801
     802class HelpOrderTests(unittest.TestCase):
     803    """
     804    Tests for the --help-orders flag.
     805    """
     806
     807
     808    def test_help_ordersPrintsSynopsisAndQuits(self):
     809        """
     810        --help-orders prints each of the available orders and then exits.
     811        """
     812
     813        self.status = None
     814        self.patch(
     815            trial.sys, "exit", lambda status: setattr(self, "status", status))
     816        self.patch(sys, "stdout", StringIO.StringIO())
     817
     818        trial.Options().parseOptions(["--help-orders"])
     819
     820        for orderName, orderDesc in trial._runOrders:
     821            match = re.search(
     822                "{0}.*{1}".format(orderName, orderDesc), sys.stdout.getvalue(),
     823            )
     824            self.assertTrue(match)
     825        self.assertEqual(self.status, 0)