Ticket #4138: sdist-support-4138.2.patch

File sdist-support-4138.2.patch, 140.1 KB (added by TimAllen, 12 years ago)

Address Glyph's review comments.

  • setup.py

    diff --git a/setup.py b/setup.py
    index 3c62eab..a2cbe80 100755
    a b def main(args): 
    4444        sys.path.insert(0, '.')
    4545    from twisted import copyright
    4646    from twisted.python.dist import (getDataFiles, getScripts, getPackages,
    47                                      setup, SDistTwisted)
     47                                     setup, _SDistTwisted)
    4848
    4949    # "" is included because core scripts are directly in bin/
    5050    projects = [''] + [x for x in os.listdir('bin')
    on event-based network programming and multiprotocol integration. 
    7474            conditionalExtensions = getExtensions(),
    7575            scripts = scripts,
    7676            data_files=getDataFiles('twisted'),
    77             cmdclass = {'sdist': SDistTwisted},
     77            cmdclass = {'sdist': _SDistTwisted},
    7878            )
    7979
    8080    if 'setuptools' in sys.modules:
  • new file twisted/python/_dist.py

    diff --git a/twisted/python/_dist.py b/twisted/python/_dist.py
    new file mode 100644
    index 0000000..f7c7ec1
    - +  
     1# -*- test-case-name: twisted.python.test.test__dist -*-
     2# Copyright (c) 2010 Twisted Matrix Laboratories.
     3# See LICENSE for details.
     4"""
     5Release-making code that distutils can safely depend on.
     6"""
     7
     8import os, fnmatch, tarfile, errno, shutil
     9from twisted.lore.scripts import lore
     10
     11
     12twisted_subprojects = ["conch", "lore", "mail", "names",
     13                       "news", "pair", "runner", "web", "web2",
     14                       "words", "vfs"]
     15
     16
     17
     18# Files and directories matching these patterns will be excluded from Twisted
     19# releases.
     20EXCLUDE_PATTERNS = ["{arch}", "_darcs", "*.py[cdo]", "*.s[ol]", ".*", "*~"]
     21
     22
     23
     24def isDistributable(filepath):
     25    """
     26    Determine if the given item should be included in Twisted distributions.
     27
     28    This function is useful for filtering out files and directories in the
     29    Twisted directory that aren't supposed to be part of the official Twisted
     30    package - things like version control system metadata, editor backup files,
     31    and various other detritus.
     32
     33    @type filepath: L{FilePath}
     34    @param filepath: The file or directory that is a candidate for packaging.
     35
     36    @rtype: C{bool}
     37    @return: True if the file should be included, False otherwise.
     38    """
     39    for pattern in EXCLUDE_PATTERNS:
     40        if fnmatch.fnmatch(filepath.basename(), pattern):
     41            return False
     42    return True
     43
     44
     45
     46class NoDocumentsFound(Exception):
     47    """
     48    Raised when no input documents are found.
     49    """
     50
     51
     52
     53class LoreBuilderMixin(object):
     54    """
     55    Base class for builders which invoke lore.
     56    """
     57    def lore(self, arguments):
     58        """
     59        Run lore with the given arguments.
     60
     61        @param arguments: A C{list} of C{str} giving command line arguments to
     62            lore which should be used.
     63        """
     64        options = lore.Options()
     65        options.parseOptions(["--null"] + arguments)
     66        lore.runGivenOptions(options)
     67
     68
     69
     70class DocBuilder(LoreBuilderMixin):
     71    """
     72    Generate HTML documentation for projects.
     73    """
     74
     75    def build(self, version, resourceDir, docDir, template, apiBaseURL=None,
     76              deleteInput=False):
     77        """
     78        Build the documentation in C{docDir} with Lore.
     79
     80        Input files ending in .xhtml will be considered. Output will written as
     81        .html files.
     82
     83        @param version: the version of the documentation to pass to lore.
     84        @type version: C{str}
     85
     86        @param resourceDir: The directory which contains the toplevel index and
     87            stylesheet file for this section of documentation.
     88        @type resourceDir: L{twisted.python.filepath.FilePath}
     89
     90        @param docDir: The directory of the documentation.
     91        @type docDir: L{twisted.python.filepath.FilePath}
     92
     93        @param template: The template used to generate the documentation.
     94        @type template: L{twisted.python.filepath.FilePath}
     95
     96        @type apiBaseURL: C{str} or C{NoneType}
     97        @param apiBaseURL: A format string which will be interpolated with the
     98            fully-qualified Python name for each API link.  For example, to
     99            generate the Twisted 8.0.0 documentation, pass
     100            C{"http://twistedmatrix.com/documents/8.0.0/api/%s.html"}.
     101
     102        @param deleteInput: If True, the input documents will be deleted after
     103            their output is generated.
     104        @type deleteInput: C{bool}
     105
     106        @raise NoDocumentsFound: When there are no .xhtml files in the given
     107            C{docDir}.
     108        """
     109        linkrel = self.getLinkrel(resourceDir, docDir)
     110        inputFiles = docDir.globChildren("*.xhtml")
     111        filenames = [x.path for x in inputFiles]
     112        if not filenames:
     113            raise NoDocumentsFound("No input documents found in %s" % (docDir,))
     114        if apiBaseURL is not None:
     115            arguments = ["--config", "baseurl=" + apiBaseURL]
     116        else:
     117            arguments = []
     118        arguments.extend(["--config", "template=%s" % (template.path,),
     119                          "--config", "ext=.html",
     120                          "--config", "version=%s" % (version,),
     121                          "--linkrel", linkrel] + filenames)
     122        self.lore(arguments)
     123        if deleteInput:
     124            for inputFile in inputFiles:
     125                inputFile.remove()
     126
     127
     128    def getLinkrel(self, resourceDir, docDir):
     129        """
     130        Calculate a value appropriate for Lore's --linkrel option.
     131
     132        Lore's --linkrel option defines how to 'find' documents that are
     133        linked to from TEMPLATE files (NOT document bodies). That is, it's a
     134        prefix for links ('a' and 'link') in the template.
     135
     136        @param resourceDir: The directory which contains the toplevel index and
     137            stylesheet file for this section of documentation.
     138        @type resourceDir: L{twisted.python.filepath.FilePath}
     139
     140        @param docDir: The directory containing documents that must link to
     141            C{resourceDir}.
     142        @type docDir: L{twisted.python.filepath.FilePath}
     143        """
     144        if resourceDir != docDir:
     145            return '/'.join(filePathDelta(docDir, resourceDir)) + "/"
     146        else:
     147            return ""
     148
     149
     150
     151class ManBuilder(LoreBuilderMixin):
     152    """
     153    Generate man pages of the different existing scripts.
     154    """
     155
     156    def build(self, manDir):
     157        """
     158        Generate Lore input files from the man pages in C{manDir}.
     159
     160        Input files ending in .1 will be considered. Output will written as
     161        -man.xhtml files.
     162
     163        @param manDir: The directory of the man pages.
     164        @type manDir: L{twisted.python.filepath.FilePath}
     165
     166        @raise NoDocumentsFound: When there are no .1 files in the given
     167            C{manDir}.
     168        """
     169        inputFiles = manDir.globChildren("*.1")
     170        filenames = [x.path for x in inputFiles]
     171        if not filenames:
     172            raise NoDocumentsFound("No manual pages found in %s" % (manDir,))
     173        arguments = ["--input", "man",
     174                     "--output", "lore",
     175                     "--config", "ext=-man.xhtml"] + filenames
     176        self.lore(arguments)
     177
     178
     179
     180def _stageFile(src, dest):
     181    """
     182    Stages src at the destination path.
     183
     184    "Staging", in this case, means "creating a temporary copy to be archived".
     185    In particular, we want to preserve all the metadata of the original file,
     186    but we don't care about whether edits to the file propagate back and forth
     187    (since the staged version should never be edited). We hard-link the file if
     188    we can, otherwise we copy it and preserve metadata.
     189
     190    @type src: L{twisted.python.filepath.FilePath}
     191    @param src: The file or path to be staged.
     192
     193    @type dest: L{twisted.python.filepath.FilePath}
     194    @param dest: The path at which the source file should be staged. Any
     195        missing directories in this path will be created.
     196
     197    @raise OSError: If the source is a file, and the destination already
     198        exists, C{OSError} will be raised with the C{errno} attribute set to
     199        C{EEXIST}.
     200    """
     201
     202    if not isDistributable(src):
     203        # Not a file we care about, quietly skip it.
     204        return
     205
     206    if src.isfile():
     207        # Make sure the directory's parent exists.
     208        destParent = dest.parent()
     209        if not destParent.exists():
     210            destParent.makedirs()
     211
     212        # If the file already exists, raise an exception.
     213        # os.link raises OSError, shutil.copy (sometimes) raises IOError or
     214        # overwrites the destination, so let's do the checking ourselves and
     215        # raise our own error.
     216        if dest.exists():
     217            raise OSError(errno.EEXIST, "File exists: %s" % (dest.path,))
     218
     219        # If we can create a hard link, that's faster than trying to copy
     220        # things around.
     221        if hasattr(os, "link"):
     222            copyfunc = os.link
     223        else:
     224            copyfunc = shutil.copy2
     225
     226        try:
     227            copyfunc(src.path, dest.path)
     228        except OSError, e:
     229            if e.errno == errno.EXDEV:
     230                shutil.copy2(src.path, dest.path)
     231            else:
     232                raise
     233
     234    elif src.isdir():
     235        if not dest.exists():
     236            dest.makedirs()
     237
     238        for child in src.children():
     239            _stageFile(child, dest.child(child.basename()))
     240
     241    else:
     242        raise NotImplementedError("Can only stage files or directories")
     243
     244
     245
     246class DistributionBuilder(object):
     247    """
     248    A builder of Twisted distributions.
     249
     250    This knows how to build tarballs for Twisted and all of its subprojects.
     251
     252    @type blacklist: C{list} of C{str}
     253    @cvar blacklist: The list subproject names to exclude from the main Twisted
     254        tarball and for which no individual project tarballs will be built.
     255    """
     256
     257    blacklist = ["vfs", "web2"]
     258
     259    def __init__(self, rootDirectory, outputDirectory, apiBaseURL=None):
     260        """
     261        Create a distribution builder.
     262
     263        @param rootDirectory: root of a Twisted export which will populate
     264            subsequent tarballs.
     265        @type rootDirectory: L{FilePath}.
     266
     267        @param outputDirectory: The directory in which to create the tarballs.
     268        @type outputDirectory: L{FilePath}
     269
     270        @type apiBaseURL: C{str} or C{NoneType}
     271        @param apiBaseURL: A format string which will be interpolated with the
     272            fully-qualified Python name for each API link.  For example, to
     273            generate the Twisted 8.0.0 documentation, pass
     274            C{"http://twistedmatrix.com/documents/8.0.0/api/%s.html"}.
     275        """
     276        self.rootDirectory = rootDirectory
     277        self.outputDirectory = outputDirectory
     278        self.apiBaseURL = apiBaseURL
     279        self.manBuilder = ManBuilder()
     280        self.docBuilder = DocBuilder()
     281
     282
     283    def _buildDocInDir(self, path, version, howtoPath):
     284        """
     285        Generate documentation in the given path, building man pages first if
     286        necessary and swallowing errors (so that directories without lore
     287        documentation in them are ignored).
     288
     289        @param path: The path containing documentation to build.
     290        @type path: L{FilePath}
     291        @param version: The version of the project to include in all generated
     292            pages.
     293        @type version: C{str}
     294        @param howtoPath: The "resource path" as L{DocBuilder} describes it.
     295        @type howtoPath: L{FilePath}
     296        """
     297        templatePath = self.rootDirectory.child("doc").child("core"
     298            ).child("howto").child("template.tpl")
     299        if path.basename() == "man":
     300            self.manBuilder.build(path)
     301        if path.isdir():
     302            try:
     303                self.docBuilder.build(version, howtoPath, path,
     304                    templatePath, self.apiBaseURL, True)
     305            except NoDocumentsFound:
     306                pass
     307
     308
     309    def buildTwistedFiles(self, version, releaseName):
     310        """
     311        Build a directory containing the main Twisted distribution.
     312        """
     313        # Make all the directories we'll need for copying things to.
     314        distDirectory = self.outputDirectory.child(releaseName)
     315        distBin = distDirectory.child("bin")
     316        distTwisted = distDirectory.child("twisted")
     317        distPlugins = distTwisted.child("plugins")
     318        distDoc = distDirectory.child("doc")
     319
     320        for dir in (distDirectory, distBin, distTwisted, distPlugins, distDoc):
     321            dir.makedirs()
     322
     323        # Now, this part is nasty.  We need to exclude blacklisted subprojects
     324        # from the main Twisted distribution. This means we need to exclude
     325        # their bin directories, their documentation directories, their
     326        # plugins, and their python packages. Given that there's no "add all
     327        # but exclude these particular paths" functionality in tarfile, we have
     328        # to walk through all these directories and add things that *aren't*
     329        # part of the blacklisted projects.
     330
     331        for binthing in self.rootDirectory.child("bin").children():
     332            # bin/admin should also not be included.
     333            if binthing.basename() not in self.blacklist + ["admin"]:
     334                _stageFile(binthing, distBin.child(binthing.basename()))
     335
     336        bad_plugins = ["twisted_%s.py" % (blacklisted,)
     337                       for blacklisted in self.blacklist]
     338
     339        for submodule in self.rootDirectory.child("twisted").children():
     340            if submodule.basename() == "plugins":
     341                for plugin in submodule.children():
     342                    if plugin.basename() not in bad_plugins:
     343                        _stageFile(plugin,
     344                                distPlugins.child(plugin.basename()))
     345            elif submodule.basename() not in self.blacklist:
     346                _stageFile(submodule, distTwisted.child(submodule.basename()))
     347
     348        for docDir in self.rootDirectory.child("doc").children():
     349            if docDir.basename() not in self.blacklist:
     350                _stageFile(docDir, distDoc.child(docDir.basename()))
     351
     352        for toplevel in self.rootDirectory.children():
     353            if not toplevel.isdir():
     354                _stageFile(toplevel, distDirectory.child(toplevel.basename()))
     355
     356        # Generate docs in the distribution directory.
     357        docPath = distDirectory.child("doc")
     358        if docPath.isdir():
     359            for subProjectDir in docPath.children():
     360                if (subProjectDir.isdir()
     361                    and subProjectDir.basename() not in self.blacklist):
     362                    for child in subProjectDir.walk():
     363                        self._buildDocInDir(child, version,
     364                            subProjectDir.child("howto"))
     365
     366
     367    def buildTwisted(self, version):
     368        """
     369        Build the main Twisted distribution in C{Twisted-<version>.tar.bz2}.
     370
     371        Projects listed in in L{blacklist} will not have their plugins, code,
     372        documentation, or bin directories included.
     373
     374        bin/admin is also excluded.
     375
     376        @type version: C{str}
     377        @param version: The version of Twisted to build.
     378
     379        @return: The tarball file.
     380        @rtype: L{FilePath}.
     381        """
     382        releaseName = "Twisted-%s" % (version,)
     383
     384        outputTree = self.outputDirectory.child(releaseName)
     385        outputFile = self.outputDirectory.child(releaseName + ".tar.bz2")
     386
     387        tarball = tarfile.TarFile.open(outputFile.path, 'w:bz2')
     388        self.buildTwistedFiles(version, releaseName)
     389        tarball.add(outputTree.path, releaseName)
     390        tarball.close()
     391
     392        outputTree.remove()
     393
     394        return outputFile
     395
     396
     397    def buildCore(self, version):
     398        """
     399        Build a core distribution in C{TwistedCore-<version>.tar.bz2}.
     400
     401        This is very similar to L{buildSubProject}, but core tarballs and the
     402        input are laid out slightly differently.
     403
     404         - scripts are in the top level of the C{bin} directory.
     405         - code is included directly from the C{twisted} directory, excluding
     406           subprojects.
     407         - all plugins except the subproject plugins are included.
     408
     409        @type version: C{str}
     410        @param version: The version of Twisted to build.
     411
     412        @return: The tarball file.
     413        @rtype: L{FilePath}.
     414        """
     415        releaseName = "TwistedCore-%s" % (version,)
     416        outputFile = self.outputDirectory.child(releaseName + ".tar.bz2")
     417        buildPath = lambda *args: '/'.join((releaseName,) + args)
     418        tarball = self._createBasicSubprojectTarball(
     419            "core", version, outputFile)
     420
     421        # Include the bin directory for the subproject.
     422        for path in self.rootDirectory.child("bin").children():
     423            if not path.isdir():
     424                tarball.add(path.path, buildPath("bin", path.basename()))
     425
     426        # Include all files within twisted/ that aren't part of a subproject.
     427        for path in self.rootDirectory.child("twisted").children():
     428            if path.basename() == "plugins":
     429                for plugin in path.children():
     430                    for subproject in twisted_subprojects:
     431                        if plugin.basename() == "twisted_%s.py" % (subproject,):
     432                            break
     433                    else:
     434                        tarball.add(plugin.path,
     435                                    buildPath("twisted", "plugins",
     436                                              plugin.basename()))
     437            elif not path.basename() in twisted_subprojects + ["topfiles"]:
     438                tarball.add(path.path, buildPath("twisted", path.basename()))
     439
     440        tarball.add(self.rootDirectory.child("twisted").child("topfiles").path,
     441                    releaseName)
     442        tarball.close()
     443
     444        return outputFile
     445
     446
     447    def buildSubProject(self, projectName, version):
     448        """
     449        Build a subproject distribution in
     450        C{Twisted<Projectname>-<version>.tar.bz2}.
     451
     452        @type projectName: C{str}
     453        @param projectName: The lowercase name of the subproject to build.
     454        @type version: C{str}
     455        @param version: The version of Twisted to build.
     456
     457        @return: The tarball file.
     458        @rtype: L{FilePath}.
     459        """
     460        releaseName = "Twisted%s-%s" % (projectName.capitalize(), version)
     461        outputFile = self.outputDirectory.child(releaseName + ".tar.bz2")
     462        buildPath = lambda *args: '/'.join((releaseName,) + args)
     463        subProjectDir = self.rootDirectory.child("twisted").child(projectName)
     464
     465        tarball = self._createBasicSubprojectTarball(projectName, version,
     466                                                     outputFile)
     467
     468        tarball.add(subProjectDir.child("topfiles").path, releaseName)
     469
     470        # Include all files in the subproject package except for topfiles.
     471        for child in subProjectDir.children():
     472            name = child.basename()
     473            if name != "topfiles":
     474                tarball.add(
     475                    child.path,
     476                    buildPath("twisted", projectName, name))
     477
     478        pluginsDir = self.rootDirectory.child("twisted").child("plugins")
     479        # Include the plugin for the subproject.
     480        pluginFileName = "twisted_%s.py" % (projectName,)
     481        pluginFile = pluginsDir.child(pluginFileName)
     482        if pluginFile.exists():
     483            tarball.add(pluginFile.path,
     484                        buildPath("twisted", "plugins", pluginFileName))
     485
     486        # Include the bin directory for the subproject.
     487        binPath = self.rootDirectory.child("bin").child(projectName)
     488        if binPath.isdir():
     489            tarball.add(binPath.path, buildPath("bin"))
     490        tarball.close()
     491
     492        return outputFile
     493
     494
     495    def _createBasicSubprojectTarball(self, projectName, version, outputFile):
     496        """
     497        Helper method to create and fill a tarball with things common between
     498        subprojects and core.
     499
     500        @param projectName: The subproject's name.
     501        @type projectName: C{str}
     502        @param version: The version of the release.
     503        @type version: C{str}
     504        @param outputFile: The location of the tar file to create.
     505        @type outputFile: L{FilePath}
     506        """
     507        releaseName = "Twisted%s-%s" % (projectName.capitalize(), version)
     508        buildPath = lambda *args: '/'.join((releaseName,) + args)
     509
     510        tarball = tarfile.TarFile.open(outputFile.path, 'w:bz2')
     511
     512        tarball.add(self.rootDirectory.child("LICENSE").path,
     513                    buildPath("LICENSE"))
     514
     515        docPath = self.rootDirectory.child("doc").child(projectName)
     516
     517        if docPath.isdir():
     518            for child in docPath.walk():
     519                self._buildDocInDir(child, version, docPath.child("howto"))
     520            tarball.add(docPath.path, buildPath("doc"))
     521
     522        return tarball
     523
     524
     525
     526def makeAPIBaseURL(version):
     527    """
     528    Guess where the Twisted API docs for a given version will live.
     529
     530    @type version: C{str}
     531    @param version: A URL-safe string containing a version number, such as
     532        "10.0.0".
     533    @rtype: C{str}
     534    @return: A URL template pointing to the Twisted API docs for the given
     535        version, ready to have the class, module or function name substituted
     536        in.
     537    """
     538    return "http://twistedmatrix.com/documents/%s/api/%%s.html" % (version,)
     539
     540
     541
     542def filePathDelta(origin, destination):
     543    """
     544    Return a list of strings that represent C{destination} as a path relative
     545    to C{origin}.
     546
     547    It is assumed that both paths represent directories, not files. That is to
     548    say, the delta of L{twisted.python.filepath.FilePath} /foo/bar to
     549    L{twisted.python.filepath.FilePath} /foo/baz will be C{../baz},
     550    not C{baz}.
     551
     552    @type origin: L{twisted.python.filepath.FilePath}
     553    @param origin: The origin of the relative path.
     554
     555    @type destination: L{twisted.python.filepath.FilePath}
     556    @param destination: The destination of the relative path.
     557    """
     558    commonItems = 0
     559    path1 = origin.path.split(os.sep)
     560    path2 = destination.path.split(os.sep)
     561    for elem1, elem2 in zip(path1, path2):
     562        if elem1 == elem2:
     563            commonItems += 1
     564    path = [".."] * (len(path1) - commonItems)
     565    return path + path2[commonItems:]
     566
     567
     568
  • twisted/python/_release.py

    diff --git a/twisted/python/_release.py b/twisted/python/_release.py
    index ef84ade..74ffbb1 100644
    a b import textwrap 
    1616from datetime import date
    1717import sys
    1818import os
    19 import shutil
    20 import errno
    2119from tempfile import mkdtemp
    22 import tarfile
    23 import fnmatch
    2420
    2521# Popen4 isn't available on Windows.  BookBuilder won't work on Windows, but
    2622# we don't care. -exarkun
    except ImportError: 
    3127
    3228from twisted.python.versions import Version
    3329from twisted.python.filepath import FilePath
    34 
    35 # This import is an example of why you shouldn't use this module unless you're
    36 # radix
    37 try:
    38     from twisted.lore.scripts import lore
    39 except ImportError:
    40     pass
     30from twisted.python._dist import LoreBuilderMixin, DistributionBuilder
     31from twisted.python._dist import makeAPIBaseURL, twisted_subprojects
    4132
    4233# The offset between a year and the corresponding major version number.
    4334VERSION_OFFSET = 2000
    4435
    4536
    46 twisted_subprojects = ["conch", "lore", "mail", "names",
    47                        "news", "pair", "runner", "web", "web2",
    48                        "words", "vfs"]
    49 
    50 
    51 # Files and directories matching these patterns will be excluded from Twisted
    52 # releases.
    53 EXCLUDE_PATTERNS = ["{arch}", "CVS", "_darcs", "RCS", "SCCS", "*.py[cdo]",
    54         "*.s[ol]", ".*", "*~"]
    55 
    56 
    57 
    58 def isDistributable(filepath):
    59     """
    60     Determine if the given item should be included in Twisted distributions.
    61 
    62     This function is useful for filtering out files and directories in the
    63     Twisted directory that aren't supposed to be part of the official Twisted
    64     package - things like version control system metadata, editor backup files,
    65     and various other detritus.
    66 
    67     @type filepath: L{FilePath}
    68     @param filepath: The file or directory that is a candidate for packaging.
    69 
    70     @rtype: C{bool}
    71     @return: True if the file should be included, False otherwise.
    72     """
    73     for pattern in EXCLUDE_PATTERNS:
    74         if fnmatch.fnmatch(filepath.basename(), pattern):
    75             return False
    76     return True
    77 
    78 
    79 
    8037def runCommand(args):
    8138    """
    8239    Execute a vector of arguments.
    def replaceInFile(filename, oldToNew): 
    303260
    304261
    305262
    306 class NoDocumentsFound(Exception):
    307     """
    308     Raised when no input documents are found.
    309     """
    310 
    311 
    312 
    313 class LoreBuilderMixin(object):
    314     """
    315     Base class for builders which invoke lore.
    316     """
    317     def lore(self, arguments):
    318         """
    319         Run lore with the given arguments.
    320 
    321         @param arguments: A C{list} of C{str} giving command line arguments to
    322             lore which should be used.
    323         """
    324         options = lore.Options()
    325         options.parseOptions(["--null"] + arguments)
    326         lore.runGivenOptions(options)
    327 
    328 
    329 
    330 class DocBuilder(LoreBuilderMixin):
    331     """
    332     Generate HTML documentation for projects.
    333     """
    334 
    335     def build(self, version, resourceDir, docDir, template, apiBaseURL=None,
    336               deleteInput=False):
    337         """
    338         Build the documentation in C{docDir} with Lore.
    339 
    340         Input files ending in .xhtml will be considered. Output will written as
    341         .html files.
    342 
    343         @param version: the version of the documentation to pass to lore.
    344         @type version: C{str}
    345 
    346         @param resourceDir: The directory which contains the toplevel index and
    347             stylesheet file for this section of documentation.
    348         @type resourceDir: L{twisted.python.filepath.FilePath}
    349 
    350         @param docDir: The directory of the documentation.
    351         @type docDir: L{twisted.python.filepath.FilePath}
    352 
    353         @param template: The template used to generate the documentation.
    354         @type template: L{twisted.python.filepath.FilePath}
    355 
    356         @type apiBaseURL: C{str} or C{NoneType}
    357         @param apiBaseURL: A format string which will be interpolated with the
    358             fully-qualified Python name for each API link.  For example, to
    359             generate the Twisted 8.0.0 documentation, pass
    360             C{"http://twistedmatrix.com/documents/8.0.0/api/%s.html"}.
    361 
    362         @param deleteInput: If True, the input documents will be deleted after
    363             their output is generated.
    364         @type deleteInput: C{bool}
    365 
    366         @raise NoDocumentsFound: When there are no .xhtml files in the given
    367             C{docDir}.
    368         """
    369         linkrel = self.getLinkrel(resourceDir, docDir)
    370         inputFiles = docDir.globChildren("*.xhtml")
    371         filenames = [x.path for x in inputFiles]
    372         if not filenames:
    373             raise NoDocumentsFound("No input documents found in %s" % (docDir,))
    374         if apiBaseURL is not None:
    375             arguments = ["--config", "baseurl=" + apiBaseURL]
    376         else:
    377             arguments = []
    378         arguments.extend(["--config", "template=%s" % (template.path,),
    379                           "--config", "ext=.html",
    380                           "--config", "version=%s" % (version,),
    381                           "--linkrel", linkrel] + filenames)
    382         self.lore(arguments)
    383         if deleteInput:
    384             for inputFile in inputFiles:
    385                 inputFile.remove()
    386 
    387 
    388     def getLinkrel(self, resourceDir, docDir):
    389         """
    390         Calculate a value appropriate for Lore's --linkrel option.
    391 
    392         Lore's --linkrel option defines how to 'find' documents that are
    393         linked to from TEMPLATE files (NOT document bodies). That is, it's a
    394         prefix for links ('a' and 'link') in the template.
    395 
    396         @param resourceDir: The directory which contains the toplevel index and
    397             stylesheet file for this section of documentation.
    398         @type resourceDir: L{twisted.python.filepath.FilePath}
    399 
    400         @param docDir: The directory containing documents that must link to
    401             C{resourceDir}.
    402         @type docDir: L{twisted.python.filepath.FilePath}
    403         """
    404         if resourceDir != docDir:
    405             return '/'.join(filePathDelta(docDir, resourceDir)) + "/"
    406         else:
    407             return ""
    408 
    409 
    410 
    411 class ManBuilder(LoreBuilderMixin):
    412     """
    413     Generate man pages of the different existing scripts.
    414     """
    415 
    416     def build(self, manDir):
    417         """
    418         Generate Lore input files from the man pages in C{manDir}.
    419 
    420         Input files ending in .1 will be considered. Output will written as
    421         -man.xhtml files.
    422 
    423         @param manDir: The directory of the man pages.
    424         @type manDir: L{twisted.python.filepath.FilePath}
    425 
    426         @raise NoDocumentsFound: When there are no .1 files in the given
    427             C{manDir}.
    428         """
    429         inputFiles = manDir.globChildren("*.1")
    430         filenames = [x.path for x in inputFiles]
    431         if not filenames:
    432             raise NoDocumentsFound("No manual pages found in %s" % (manDir,))
    433         arguments = ["--input", "man",
    434                      "--output", "lore",
    435                      "--config", "ext=-man.xhtml"] + filenames
    436         self.lore(arguments)
    437 
    438 
    439 
    440263class APIBuilder(object):
    441264    """
    442265    Generate API documentation from source files using
    class NewsBuilder(object): 
    842665
    843666
    844667
    845 def filePathDelta(origin, destination):
    846     """
    847     Return a list of strings that represent C{destination} as a path relative
    848     to C{origin}.
    849 
    850     It is assumed that both paths represent directories, not files. That is to
    851     say, the delta of L{twisted.python.filepath.FilePath} /foo/bar to
    852     L{twisted.python.filepath.FilePath} /foo/baz will be C{../baz},
    853     not C{baz}.
    854 
    855     @type origin: L{twisted.python.filepath.FilePath}
    856     @param origin: The origin of the relative path.
    857 
    858     @type destination: L{twisted.python.filepath.FilePath}
    859     @param destination: The destination of the relative path.
    860     """
    861     commonItems = 0
    862     path1 = origin.path.split(os.sep)
    863     path2 = destination.path.split(os.sep)
    864     for elem1, elem2 in zip(path1, path2):
    865         if elem1 == elem2:
    866             commonItems += 1
    867     path = [".."] * (len(path1) - commonItems)
    868     return path + path2[commonItems:]
    869 
    870 
    871 
    872 def _stageFile(src, dest):
    873     """
    874     Stages src at the destination path.
    875 
    876     "Staging", in this case, means "creating a temporary copy to be archived".
    877     In particular, we want to preserve all the metadata of the original file,
    878     but we don't care about whether edits to the file propagate back and forth
    879     (since the staged version should never be edited). We hard-link the file if
    880     we can, otherwise we copy it and preserve metadata.
    881 
    882     @type src: L{twisted.python.filepath.FilePath}
    883     @param src: The file or path to be staged.
    884 
    885     @type dest: L{twisted.python.filepath.FilePath}
    886     @param dest: The path at which the source file should be staged. Any
    887         missing directories in this path will be created.
    888 
    889     @raise OSError: If the source is a file, and the destination already
    890         exists, C{OSError} will be raised with the C{errno} attribute set to
    891         C{EEXIST}.
    892     """
    893 
    894     if not isDistributable(src):
    895         # Not a file we care about, quietly skip it.
    896         return
    897 
    898     if src.isfile():
    899         # Make sure the directory's parent exists.
    900         destParent = dest.parent()
    901         if not destParent.exists():
    902             destParent.makedirs()
    903 
    904         # If the file already exists, raise an exception.
    905         # os.link raises OSError, shutil.copy (sometimes) raises IOError or
    906         # overwrites the destination, so let's do the checking ourselves and
    907         # raise our own error.
    908         if dest.exists():
    909             raise OSError(errno.EEXIST, "File exists: %s" % (dest.path,))
    910 
    911         # If we can create a hard link, that's faster than trying to copy
    912         # things around.
    913         if hasattr(os, "link"):
    914             copyfunc = os.link
    915         else:
    916             copyfunc = shutil.copy2
    917 
    918         try:
    919             copyfunc(src.path, dest.path)
    920         except OSError, e:
    921             if e.errno == errno.EXDEV:
    922                 shutil.copy2(src.path, dest.path)
    923             else:
    924                 raise
    925 
    926     elif src.isdir():
    927         if not dest.exists():
    928             dest.makedirs()
    929 
    930         for child in src.children():
    931             _stageFile(child, dest.child(child.basename()))
    932 
    933     else:
    934         raise NotImplementedError("Can only stage files or directories")
    935 
    936 
    937 
    938 class DistributionBuilder(object):
    939     """
    940     A builder of Twisted distributions.
    941 
    942     This knows how to build tarballs for Twisted and all of its subprojects.
    943 
    944     @type blacklist: C{list} of C{str}
    945     @cvar blacklist: The list subproject names to exclude from the main Twisted
    946         tarball and for which no individual project tarballs will be built.
    947     """
    948 
    949     blacklist = ["vfs", "web2"]
    950 
    951     def __init__(self, rootDirectory, outputDirectory, apiBaseURL=None):
    952         """
    953         Create a distribution builder.
    954 
    955         @param rootDirectory: root of a Twisted export which will populate
    956             subsequent tarballs.
    957         @type rootDirectory: L{FilePath}.
    958 
    959         @param outputDirectory: The directory in which to create the tarballs.
    960         @type outputDirectory: L{FilePath}
    961 
    962         @type apiBaseURL: C{str} or C{NoneType}
    963         @param apiBaseURL: A format string which will be interpolated with the
    964             fully-qualified Python name for each API link.  For example, to
    965             generate the Twisted 8.0.0 documentation, pass
    966             C{"http://twistedmatrix.com/documents/8.0.0/api/%s.html"}.
    967         """
    968         self.rootDirectory = rootDirectory
    969         self.outputDirectory = outputDirectory
    970         self.apiBaseURL = apiBaseURL
    971         self.manBuilder = ManBuilder()
    972         self.docBuilder = DocBuilder()
    973 
    974 
    975     def _buildDocInDir(self, path, version, howtoPath):
    976         """
    977         Generate documentation in the given path, building man pages first if
    978         necessary and swallowing errors (so that directories without lore
    979         documentation in them are ignored).
    980 
    981         @param path: The path containing documentation to build.
    982         @type path: L{FilePath}
    983         @param version: The version of the project to include in all generated
    984             pages.
    985         @type version: C{str}
    986         @param howtoPath: The "resource path" as L{DocBuilder} describes it.
    987         @type howtoPath: L{FilePath}
    988         """
    989         templatePath = self.rootDirectory.child("doc").child("core"
    990             ).child("howto").child("template.tpl")
    991         if path.basename() == "man":
    992             self.manBuilder.build(path)
    993         if path.isdir():
    994             try:
    995                 self.docBuilder.build(version, howtoPath, path,
    996                     templatePath, self.apiBaseURL, True)
    997             except NoDocumentsFound:
    998                 pass
    999 
    1000 
    1001     def buildTwistedFiles(self, version, releaseName):
    1002         """
    1003         Build a directory containing the main Twisted distribution.
    1004         """
    1005         # Make all the directories we'll need for copying things to.
    1006         distDirectory = self.outputDirectory.child(releaseName)
    1007         distBin = distDirectory.child("bin")
    1008         distTwisted = distDirectory.child("twisted")
    1009         distPlugins = distTwisted.child("plugins")
    1010         distDoc = distDirectory.child("doc")
    1011 
    1012         for dir in (distDirectory, distBin, distTwisted, distPlugins, distDoc):
    1013             dir.makedirs()
    1014 
    1015         # Now, this part is nasty.  We need to exclude blacklisted subprojects
    1016         # from the main Twisted distribution. This means we need to exclude
    1017         # their bin directories, their documentation directories, their
    1018         # plugins, and their python packages. Given that there's no "add all
    1019         # but exclude these particular paths" functionality in tarfile, we have
    1020         # to walk through all these directories and add things that *aren't*
    1021         # part of the blacklisted projects.
    1022 
    1023         for binthing in self.rootDirectory.child("bin").children():
    1024             # bin/admin should also not be included.
    1025             if binthing.basename() not in self.blacklist + ["admin"]:
    1026                 _stageFile(binthing, distBin.child(binthing.basename()))
    1027 
    1028         bad_plugins = ["twisted_%s.py" % (blacklisted,)
    1029                        for blacklisted in self.blacklist]
    1030 
    1031         for submodule in self.rootDirectory.child("twisted").children():
    1032             if submodule.basename() == "plugins":
    1033                 for plugin in submodule.children():
    1034                     if plugin.basename() not in bad_plugins:
    1035                         _stageFile(plugin,
    1036                                 distPlugins.child(plugin.basename()))
    1037             elif submodule.basename() not in self.blacklist:
    1038                 _stageFile(submodule, distTwisted.child(submodule.basename()))
    1039 
    1040         for docDir in self.rootDirectory.child("doc").children():
    1041             if docDir.basename() not in self.blacklist:
    1042                 _stageFile(docDir, distDoc.child(docDir.basename()))
    1043 
    1044         for toplevel in self.rootDirectory.children():
    1045             if not toplevel.isdir():
    1046                 _stageFile(toplevel, distDirectory.child(toplevel.basename()))
    1047 
    1048         # Generate docs in the distribution directory.
    1049         docPath = distDirectory.child("doc")
    1050         if docPath.isdir():
    1051             for subProjectDir in docPath.children():
    1052                 if (subProjectDir.isdir()
    1053                     and subProjectDir.basename() not in self.blacklist):
    1054                     for child in subProjectDir.walk():
    1055                         self._buildDocInDir(child, version,
    1056                             subProjectDir.child("howto"))
    1057 
    1058 
    1059     def buildTwisted(self, version):
    1060         """
    1061         Build the main Twisted distribution in C{Twisted-<version>.tar.bz2}.
    1062 
    1063         Projects listed in in L{blacklist} will not have their plugins, code,
    1064         documentation, or bin directories included.
    1065 
    1066         bin/admin is also excluded.
    1067 
    1068         @type version: C{str}
    1069         @param version: The version of Twisted to build.
    1070 
    1071         @return: The tarball file.
    1072         @rtype: L{FilePath}.
    1073         """
    1074         releaseName = "Twisted-%s" % (version,)
    1075 
    1076         outputTree = self.outputDirectory.child(releaseName)
    1077         outputFile = self.outputDirectory.child(releaseName + ".tar.bz2")
    1078 
    1079         tarball = tarfile.TarFile.open(outputFile.path, 'w:bz2')
    1080         self.buildTwistedFiles(version, releaseName)
    1081         tarball.add(outputTree.path, releaseName)
    1082         tarball.close()
    1083 
    1084         outputTree.remove()
    1085 
    1086         return outputFile
    1087 
    1088 
    1089     def buildCore(self, version):
    1090         """
    1091         Build a core distribution in C{TwistedCore-<version>.tar.bz2}.
    1092 
    1093         This is very similar to L{buildSubProject}, but core tarballs and the
    1094         input are laid out slightly differently.
    1095 
    1096          - scripts are in the top level of the C{bin} directory.
    1097          - code is included directly from the C{twisted} directory, excluding
    1098            subprojects.
    1099          - all plugins except the subproject plugins are included.
    1100 
    1101         @type version: C{str}
    1102         @param version: The version of Twisted to build.
    1103 
    1104         @return: The tarball file.
    1105         @rtype: L{FilePath}.
    1106         """
    1107         releaseName = "TwistedCore-%s" % (version,)
    1108         outputFile = self.outputDirectory.child(releaseName + ".tar.bz2")
    1109         buildPath = lambda *args: '/'.join((releaseName,) + args)
    1110         tarball = self._createBasicSubprojectTarball(
    1111             "core", version, outputFile)
    1112 
    1113         # Include the bin directory for the subproject.
    1114         for path in self.rootDirectory.child("bin").children():
    1115             if not path.isdir():
    1116                 tarball.add(path.path, buildPath("bin", path.basename()))
    1117 
    1118         # Include all files within twisted/ that aren't part of a subproject.
    1119         for path in self.rootDirectory.child("twisted").children():
    1120             if path.basename() == "plugins":
    1121                 for plugin in path.children():
    1122                     for subproject in twisted_subprojects:
    1123                         if plugin.basename() == "twisted_%s.py" % (subproject,):
    1124                             break
    1125                     else:
    1126                         tarball.add(plugin.path,
    1127                                     buildPath("twisted", "plugins",
    1128                                               plugin.basename()))
    1129             elif not path.basename() in twisted_subprojects + ["topfiles"]:
    1130                 tarball.add(path.path, buildPath("twisted", path.basename()))
    1131 
    1132         tarball.add(self.rootDirectory.child("twisted").child("topfiles").path,
    1133                     releaseName)
    1134         tarball.close()
    1135 
    1136         return outputFile
    1137 
    1138 
    1139     def buildSubProject(self, projectName, version):
    1140         """
    1141         Build a subproject distribution in
    1142         C{Twisted<Projectname>-<version>.tar.bz2}.
    1143 
    1144         @type projectName: C{str}
    1145         @param projectName: The lowercase name of the subproject to build.
    1146         @type version: C{str}
    1147         @param version: The version of Twisted to build.
    1148 
    1149         @return: The tarball file.
    1150         @rtype: L{FilePath}.
    1151         """
    1152         releaseName = "Twisted%s-%s" % (projectName.capitalize(), version)
    1153         outputFile = self.outputDirectory.child(releaseName + ".tar.bz2")
    1154         buildPath = lambda *args: '/'.join((releaseName,) + args)
    1155         subProjectDir = self.rootDirectory.child("twisted").child(projectName)
    1156 
    1157         tarball = self._createBasicSubprojectTarball(projectName, version,
    1158                                                      outputFile)
    1159 
    1160         tarball.add(subProjectDir.child("topfiles").path, releaseName)
    1161 
    1162         # Include all files in the subproject package except for topfiles.
    1163         for child in subProjectDir.children():
    1164             name = child.basename()
    1165             if name != "topfiles":
    1166                 tarball.add(
    1167                     child.path,
    1168                     buildPath("twisted", projectName, name))
    1169 
    1170         pluginsDir = self.rootDirectory.child("twisted").child("plugins")
    1171         # Include the plugin for the subproject.
    1172         pluginFileName = "twisted_%s.py" % (projectName,)
    1173         pluginFile = pluginsDir.child(pluginFileName)
    1174         if pluginFile.exists():
    1175             tarball.add(pluginFile.path,
    1176                         buildPath("twisted", "plugins", pluginFileName))
    1177 
    1178         # Include the bin directory for the subproject.
    1179         binPath = self.rootDirectory.child("bin").child(projectName)
    1180         if binPath.isdir():
    1181             tarball.add(binPath.path, buildPath("bin"))
    1182         tarball.close()
    1183 
    1184         return outputFile
    1185 
    1186 
    1187     def _createBasicSubprojectTarball(self, projectName, version, outputFile):
    1188         """
    1189         Helper method to create and fill a tarball with things common between
    1190         subprojects and core.
    1191 
    1192         @param projectName: The subproject's name.
    1193         @type projectName: C{str}
    1194         @param version: The version of the release.
    1195         @type version: C{str}
    1196         @param outputFile: The location of the tar file to create.
    1197         @type outputFile: L{FilePath}
    1198         """
    1199         releaseName = "Twisted%s-%s" % (projectName.capitalize(), version)
    1200         buildPath = lambda *args: '/'.join((releaseName,) + args)
    1201 
    1202         tarball = tarfile.TarFile.open(outputFile.path, 'w:bz2')
    1203 
    1204         tarball.add(self.rootDirectory.child("LICENSE").path,
    1205                     buildPath("LICENSE"))
    1206 
    1207         docPath = self.rootDirectory.child("doc").child(projectName)
    1208 
    1209         if docPath.isdir():
    1210             for child in docPath.walk():
    1211                 self._buildDocInDir(child, version, docPath.child("howto"))
    1212             tarball.add(docPath.path, buildPath("doc"))
    1213 
    1214         return tarball
    1215 
    1216 
    1217 
    1218668class UncleanWorkingDirectory(Exception):
    1219669    """
    1220670    Raised when the working directory of an SVN checkout is unclean.
    class NotWorkingDirectory(Exception): 
    1229679
    1230680
    1231681
    1232 def makeAPIBaseURL(version):
    1233     """
    1234     Guess where the Twisted API docs for a given version will live.
    1235 
    1236     @type version: C{str}
    1237     @param version: A URL-safe string containing a version number, such as
    1238         "10.0.0".
    1239     @rtype: C{str}
    1240     @return: A URL template pointing to the Twisted API docs for the given
    1241         version, ready to have the class, module or function name substituted
    1242         in.
    1243     """
    1244     return "http://twistedmatrix.com/documents/%s/api/%%s.html" % (version,)
    1245 
    1246 
    1247 
    1248682def buildAllTarballs(checkout, destination):
    1249683    """
    1250684    Build complete tarballs (including documentation) for Twisted and all
  • twisted/python/dist.py

    diff --git a/twisted/python/dist.py b/twisted/python/dist.py
    index 8bfb945..6d91b52 100644
    a b Don't use this outside of Twisted. 
    66Maintainer: Christopher Armstrong
    77"""
    88
    9 import sys, os
     9import os
    1010from distutils.command import (build_scripts, install_data, build_ext,
    1111                               sdist)
    1212from distutils.errors import CompileError
    1313from distutils import core
    1414from distutils.core import Extension
    1515from twisted.python.filepath import FilePath
    16 from twisted.python._release import DistributionBuilder, makeAPIBaseURL
    17 from twisted.python._release import isDistributable
     16from twisted.python._dist import DistributionBuilder, makeAPIBaseURL
     17from twisted.python._dist import isDistributable
    1818
    1919
    2020
    class build_ext_twisted(build_ext.build_ext): 
    347347        return self._compile_helper("#include <%s>\n" % header_name)
    348348
    349349
    350 class SDistTwisted(sdist.sdist):
     350class _SDistTwisted(sdist.sdist):
    351351    """
    352352    Build a Twisted source distribution like the official release scripts do.
    353353    """
    class SDistTwisted(sdist.sdist): 
    372372        rootDirectory = FilePath(".")
    373373        outputDirectory = FilePath(".")
    374374        version = self.distribution.get_version()
    375         apiBaseURL = makeAPIBaseURL(version)
    376375        builder = DistributionBuilder(rootDirectory, outputDirectory,
    377376                apiBaseURL=makeAPIBaseURL(version))
    378377        builder.buildTwistedFiles(version, basedir)
  • new file twisted/python/test/test__dist.py

    diff --git a/twisted/python/test/test__dist.py b/twisted/python/test/test__dist.py
    new file mode 100644
    index 0000000..35181f5
    - +  
     1# Copyright (c) 2010 Twisted Matrix Laboratories.
     2# See LICENSE for details.
     3
     4import os, stat, errno, tarfile
     5from xml.dom import minidom as dom
     6from twisted.trial.unittest import TestCase
     7from twisted.python.filepath import FilePath
     8
     9from twisted.python._dist import DocBuilder, ManBuilder, isDistributable
     10from twisted.python._dist import makeAPIBaseURL, DistributionBuilder
     11from twisted.python._dist import NoDocumentsFound, filePathDelta
     12from twisted.python._dist import _stageFile
     13
     14
     15
     16# When we test that scripts are installed with the "correct" permissions, we
     17# expect the "correct" permissions to be rwxr-xr-x
     18SCRIPT_PERMS = (
     19        stat.S_IRWXU # rwx
     20        | stat.S_IRGRP | stat.S_IXGRP # r-x
     21        | stat.S_IROTH | stat.S_IXOTH) # r-x
     22
     23
     24
     25# Check a bunch of dependencies to skip tests if necessary.
     26try:
     27    from twisted.lore.scripts import lore
     28except ImportError:
     29    loreSkip = "Lore is not present."
     30else:
     31    loreSkip = None
     32
     33
     34
     35class StructureAssertingMixin(object):
     36    """
     37    A mixin for L{TestCase} subclasses which provides some methods for asserting
     38    the structure and contents of directories and files on the filesystem.
     39    """
     40    def createStructure(self, parent, dirDict, origRoot=None):
     41        """
     42        Create a set of directories and files given a dict defining their
     43        structure.
     44
     45        @param parent: The directory in which to create the structure.  It must
     46            already exist.
     47        @type parent: L{FilePath}
     48
     49        @param dirDict: The dict defining the structure. Keys should be strings
     50            naming files, values should be strings describing file contents OR
     51            dicts describing subdirectories.  All files are written in binary
     52            mode.  Any string values are assumed to describe text files and
     53            will have their newlines replaced with the platform-native newline
     54            convention.  For example::
     55
     56                {"foofile": "foocontents",
     57                 "bardir": {"barfile": "bar\ncontents"}}
     58        @type dirDict: C{dict}
     59
     60        @param origRoot: The directory provided as C{parent} in the original
     61            invocation of C{createStructure}. Leave this as C{None}, it's used
     62            in recursion.
     63        @type origRoot: L{FilePath} or C{None}
     64        """
     65        if origRoot is None:
     66            origRoot = parent
     67
     68        for x in dirDict:
     69            child = parent.child(x)
     70            if isinstance(dirDict[x], dict):
     71                child.createDirectory()
     72                self.createStructure(child, dirDict[x], origRoot=origRoot)
     73
     74                # If x is in a bin directory, make sure children
     75                # representing files have the executable bit set.
     76                if "bin" in child.segmentsFrom(origRoot):
     77                    for script in [k for (k,v) in dirDict[x].items()
     78                            if isinstance(v, basestring)]:
     79                        scriptPath = child.child(script)
     80                        scriptPath.chmod(SCRIPT_PERMS)
     81
     82            else:
     83                child.setContent(dirDict[x].replace('\n', os.linesep))
     84
     85    def assertStructure(self, root, dirDict):
     86        """
     87        Assert that a directory is equivalent to one described by a dict.
     88
     89        @param root: The filesystem directory to compare.
     90        @type root: L{FilePath}
     91        @param dirDict: The dict that should describe the contents of the
     92            directory. It should be the same structure as the C{dirDict}
     93            parameter to L{createStructure}.
     94        @type dirDict: C{dict}
     95        """
     96        children = [x.basename() for x in root.children()]
     97        for x in dirDict:
     98            child = root.child(x)
     99            if isinstance(dirDict[x], dict):
     100                self.assertTrue(child.isdir(), "%s is not a dir!"
     101                                % (child.path,))
     102                self.assertStructure(child, dirDict[x])
     103
     104                # If x is in a bin directory, make sure children
     105                # representing files have the executable bit set.
     106                if "/bin" in child.path:
     107                    for script in [k for (k,v) in dirDict[x].items()
     108                            if isinstance(v, basestring)]:
     109                        scriptPath = child.child(script)
     110                        scriptPath.restat()
     111                        # What with SVN and umask and all that jazz, all we can
     112                        # really check is that these scripts have at least one
     113                        # executable bit set.
     114                        self.assertTrue(scriptPath.statinfo.st_mode &
     115                                (stat.S_IXUSR|stat.S_IXGRP|stat.S_IXOTH),
     116                                "File %r should be executable"
     117                                % (scriptPath.path,))
     118            else:
     119                a = child.getContent().replace(os.linesep, '\n')
     120                self.assertEquals(a, dirDict[x], child.path)
     121            children.remove(x)
     122        if children:
     123            self.fail("There were extra children in %s: %s"
     124                      % (root.path, children))
     125
     126
     127    def assertExtractedStructure(self, outputFile, dirDict):
     128        """
     129        Assert that a tarfile content is equivalent to one described by a dict.
     130
     131        @param outputFile: The tar file built by L{DistributionBuilder}.
     132        @type outputFile: L{FilePath}.
     133        @param dirDict: The dict that should describe the contents of the
     134            directory. It should be the same structure as the C{dirDict}
     135            parameter to L{createStructure}.
     136        @type dirDict: C{dict}
     137        """
     138        tarFile = tarfile.TarFile.open(outputFile.path, "r:bz2")
     139        extracted = FilePath(self.mktemp())
     140        extracted.createDirectory()
     141        for info in tarFile:
     142            tarFile.extract(info, path=extracted.path)
     143        self.assertStructure(extracted.children()[0], dirDict)
     144
     145
     146
     147class BuilderTestsMixin(object):
     148    """
     149    A mixin class which provides various methods for creating sample Lore input
     150    and output.
     151
     152    @cvar template: The lore template that will be used to prepare sample
     153    output.
     154    @type template: C{str}
     155
     156    @ivar docCounter: A counter which is incremented every time input is
     157        generated and which is included in the documents.
     158    @type docCounter: C{int}
     159    """
     160    template = '''
     161    <html>
     162    <head><title>Yo:</title></head>
     163    <body>
     164    <div class="body" />
     165    <a href="index.html">Index</a>
     166    <span class="version">Version: </span>
     167    </body>
     168    </html>
     169    '''
     170
     171    def setUp(self):
     172        """
     173        Initialize the doc counter which ensures documents are unique.
     174        """
     175        self.docCounter = 0
     176
     177
     178    def assertXMLEqual(self, first, second):
     179        """
     180        Verify that two strings represent the same XML document.
     181        """
     182        self.assertEqual(
     183            dom.parseString(first).toxml(),
     184            dom.parseString(second).toxml())
     185
     186
     187    def getArbitraryOutput(self, version, counter, prefix="", apiBaseURL="%s"):
     188        """
     189        Get the correct HTML output for the arbitrary input returned by
     190        L{getArbitraryLoreInput} for the given parameters.
     191
     192        @param version: The version string to include in the output.
     193        @type version: C{str}
     194        @param counter: A counter to include in the output.
     195        @type counter: C{int}
     196        """
     197        document = """\
     198<?xml version="1.0"?><html>
     199    <head><title>Yo:Hi! Title: %(count)d</title></head>
     200    <body>
     201    <div class="content">Hi! %(count)d<div class="API"><a href="%(foobarLink)s" title="foobar">foobar</a></div></div>
     202    <a href="%(prefix)sindex.html">Index</a>
     203    <span class="version">Version: %(version)s</span>
     204    </body>
     205    </html>"""
     206        # Try to normalize irrelevant whitespace.
     207        return dom.parseString(
     208            document % {"count": counter, "prefix": prefix,
     209                        "version": version,
     210                        "foobarLink": apiBaseURL % ("foobar",)}).toxml('utf-8')
     211
     212
     213    def getArbitraryLoreInput(self, counter):
     214        """
     215        Get an arbitrary, unique (for this test case) string of lore input.
     216
     217        @param counter: A counter to include in the input.
     218        @type counter: C{int}
     219        """
     220        template = (
     221            '<html>'
     222            '<head><title>Hi! Title: %(count)s</title></head>'
     223            '<body>'
     224            'Hi! %(count)s'
     225            '<div class="API">foobar</div>'
     226            '</body>'
     227            '</html>')
     228        return template % {"count": counter}
     229
     230
     231    def getArbitraryLoreInputAndOutput(self, version, prefix="",
     232                                       apiBaseURL="%s"):
     233        """
     234        Get an input document along with expected output for lore run on that
     235        output document, assuming an appropriately-specified C{self.template}.
     236
     237        @param version: A version string to include in the input and output.
     238        @type version: C{str}
     239        @param prefix: The prefix to include in the link to the index.
     240        @type prefix: C{str}
     241
     242        @return: A two-tuple of input and expected output.
     243        @rtype: C{(str, str)}.
     244        """
     245        self.docCounter += 1
     246        return (self.getArbitraryLoreInput(self.docCounter),
     247                self.getArbitraryOutput(version, self.docCounter,
     248                                        prefix=prefix, apiBaseURL=apiBaseURL))
     249
     250
     251    def getArbitraryManInput(self):
     252        """
     253        Get an arbitrary man page content.
     254        """
     255        return """.TH MANHOLE "1" "August 2001" "" ""
     256.SH NAME
     257manhole \- Connect to a Twisted Manhole service
     258.SH SYNOPSIS
     259.B manhole
     260.SH DESCRIPTION
     261manhole is a GTK interface to Twisted Manhole services. You can execute python
     262code as if at an interactive Python console inside a running Twisted process
     263with this."""
     264
     265
     266    def getArbitraryManLoreOutput(self):
     267        """
     268        Get an arbitrary lore input document which represents man-to-lore
     269        output based on the man page returned from L{getArbitraryManInput}
     270        """
     271        return """\
     272<?xml version="1.0"?>
     273<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
     274    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
     275<html><head>
     276<title>MANHOLE.1</title></head>
     277<body>
     278
     279<h1>MANHOLE.1</h1>
     280
     281<h2>NAME</h2>
     282
     283<p>manhole - Connect to a Twisted Manhole service
     284</p>
     285
     286<h2>SYNOPSIS</h2>
     287
     288<p><strong>manhole</strong> </p>
     289
     290<h2>DESCRIPTION</h2>
     291
     292<p>manhole is a GTK interface to Twisted Manhole services. You can execute python
     293code as if at an interactive Python console inside a running Twisted process
     294with this.</p>
     295
     296</body>
     297</html>
     298"""
     299
     300    def getArbitraryManHTMLOutput(self, version, prefix=""):
     301        """
     302        Get an arbitrary lore output document which represents the lore HTML
     303        output based on the input document returned from
     304        L{getArbitraryManLoreOutput}.
     305
     306        @param version: A version string to include in the document.
     307        @type version: C{str}
     308        @param prefix: The prefix to include in the link to the index.
     309        @type prefix: C{str}
     310        """
     311        # Try to normalize the XML a little bit.
     312        return dom.parseString("""\
     313<?xml version="1.0" ?><html>
     314    <head><title>Yo:MANHOLE.1</title></head>
     315    <body>
     316    <div class="content">
     317
     318<span/>
     319
     320<h2>NAME<a name="auto0"/></h2>
     321
     322<p>manhole - Connect to a Twisted Manhole service
     323</p>
     324
     325<h2>SYNOPSIS<a name="auto1"/></h2>
     326
     327<p><strong>manhole</strong> </p>
     328
     329<h2>DESCRIPTION<a name="auto2"/></h2>
     330
     331<p>manhole is a GTK interface to Twisted Manhole services. You can execute python
     332code as if at an interactive Python console inside a running Twisted process
     333with this.</p>
     334
     335</div>
     336    <a href="%(prefix)sindex.html">Index</a>
     337    <span class="version">Version: %(version)s</span>
     338    </body>
     339    </html>""" % {
     340            'prefix': prefix, 'version': version}).toxml("utf-8")
     341
     342
     343
     344class DocBuilderTestCase(TestCase, BuilderTestsMixin):
     345    """
     346    Tests for L{DocBuilder}.
     347
     348    Note for future maintainers: The exact byte equality assertions throughout
     349    this suite may need to be updated due to minor differences in lore. They
     350    should not be taken to mean that Lore must maintain the same byte format
     351    forever. Feel free to update the tests when Lore changes, but please be
     352    careful.
     353    """
     354    skip = loreSkip
     355
     356    def setUp(self):
     357        """
     358        Set up a few instance variables that will be useful.
     359
     360        @ivar builder: A plain L{DocBuilder}.
     361        @ivar docCounter: An integer to be used as a counter by the
     362            C{getArbitrary...} methods.
     363        @ivar howtoDir: A L{FilePath} representing a directory to be used for
     364            containing Lore documents.
     365        @ivar templateFile: A L{FilePath} representing a file with
     366            C{self.template} as its content.
     367        """
     368        BuilderTestsMixin.setUp(self)
     369        self.builder = DocBuilder()
     370        self.howtoDir = FilePath(self.mktemp())
     371        self.howtoDir.createDirectory()
     372        self.templateFile = self.howtoDir.child("template.tpl")
     373        self.templateFile.setContent(self.template)
     374
     375
     376    def test_build(self):
     377        """
     378        The L{DocBuilder} runs lore on all .xhtml files within a directory.
     379        """
     380        version = "1.2.3"
     381        input1, output1 = self.getArbitraryLoreInputAndOutput(version)
     382        input2, output2 = self.getArbitraryLoreInputAndOutput(version)
     383
     384        self.howtoDir.child("one.xhtml").setContent(input1)
     385        self.howtoDir.child("two.xhtml").setContent(input2)
     386
     387        self.builder.build(version, self.howtoDir, self.howtoDir,
     388                           self.templateFile)
     389        out1 = self.howtoDir.child('one.html')
     390        out2 = self.howtoDir.child('two.html')
     391        self.assertXMLEqual(out1.getContent(), output1)
     392        self.assertXMLEqual(out2.getContent(), output2)
     393
     394
     395    def test_noDocumentsFound(self):
     396        """
     397        The C{build} method raises L{NoDocumentsFound} if there are no
     398        .xhtml files in the given directory.
     399        """
     400        self.assertRaises(
     401            NoDocumentsFound,
     402            self.builder.build, "1.2.3", self.howtoDir, self.howtoDir,
     403            self.templateFile)
     404
     405
     406    def test_parentDocumentLinking(self):
     407        """
     408        The L{DocBuilder} generates correct links from documents to
     409        template-generated links like stylesheets and index backreferences.
     410        """
     411        input = self.getArbitraryLoreInput(0)
     412        tutoDir = self.howtoDir.child("tutorial")
     413        tutoDir.createDirectory()
     414        tutoDir.child("child.xhtml").setContent(input)
     415        self.builder.build("1.2.3", self.howtoDir, tutoDir, self.templateFile)
     416        outFile = tutoDir.child('child.html')
     417        self.assertIn('<a href="../index.html">Index</a>',
     418                      outFile.getContent())
     419
     420
     421    def test_siblingDirectoryDocumentLinking(self):
     422        """
     423        It is necessary to generate documentation in a directory foo/bar where
     424        stylesheet and indexes are located in foo/baz. Such resources should be
     425        appropriately linked to.
     426        """
     427        input = self.getArbitraryLoreInput(0)
     428        resourceDir = self.howtoDir.child("resources")
     429        docDir = self.howtoDir.child("docs")
     430        docDir.createDirectory()
     431        docDir.child("child.xhtml").setContent(input)
     432        self.builder.build("1.2.3", resourceDir, docDir, self.templateFile)
     433        outFile = docDir.child('child.html')
     434        self.assertIn('<a href="../resources/index.html">Index</a>',
     435                      outFile.getContent())
     436
     437
     438    def test_apiLinking(self):
     439        """
     440        The L{DocBuilder} generates correct links from documents to API
     441        documentation.
     442        """
     443        version = "1.2.3"
     444        input, output = self.getArbitraryLoreInputAndOutput(version)
     445        self.howtoDir.child("one.xhtml").setContent(input)
     446
     447        self.builder.build(version, self.howtoDir, self.howtoDir,
     448                           self.templateFile, "scheme:apilinks/%s.ext")
     449        out = self.howtoDir.child('one.html')
     450        self.assertIn(
     451            '<a href="scheme:apilinks/foobar.ext" title="foobar">foobar</a>',
     452            out.getContent())
     453
     454
     455    def test_deleteInput(self):
     456        """
     457        L{DocBuilder.build} can be instructed to delete the input files after
     458        generating the output based on them.
     459        """
     460        input1 = self.getArbitraryLoreInput(0)
     461        self.howtoDir.child("one.xhtml").setContent(input1)
     462        self.builder.build("whatever", self.howtoDir, self.howtoDir,
     463                           self.templateFile, deleteInput=True)
     464        self.assertTrue(self.howtoDir.child('one.html').exists())
     465        self.assertFalse(self.howtoDir.child('one.xhtml').exists())
     466
     467
     468    def test_doNotDeleteInput(self):
     469        """
     470        Input will not be deleted by default.
     471        """
     472        input1 = self.getArbitraryLoreInput(0)
     473        self.howtoDir.child("one.xhtml").setContent(input1)
     474        self.builder.build("whatever", self.howtoDir, self.howtoDir,
     475                           self.templateFile)
     476        self.assertTrue(self.howtoDir.child('one.html').exists())
     477        self.assertTrue(self.howtoDir.child('one.xhtml').exists())
     478
     479
     480    def test_getLinkrelToSameDirectory(self):
     481        """
     482        If the doc and resource directories are the same, the linkrel should be
     483        an empty string.
     484        """
     485        linkrel = self.builder.getLinkrel(FilePath("/foo/bar"),
     486                                          FilePath("/foo/bar"))
     487        self.assertEquals(linkrel, "")
     488
     489
     490    def test_getLinkrelToParentDirectory(self):
     491        """
     492        If the doc directory is a child of the resource directory, the linkrel
     493        should make use of '..'.
     494        """
     495        linkrel = self.builder.getLinkrel(FilePath("/foo"),
     496                                          FilePath("/foo/bar"))
     497        self.assertEquals(linkrel, "../")
     498
     499
     500    def test_getLinkrelToSibling(self):
     501        """
     502        If the doc directory is a sibling of the resource directory, the
     503        linkrel should make use of '..' and a named segment.
     504        """
     505        linkrel = self.builder.getLinkrel(FilePath("/foo/howto"),
     506                                          FilePath("/foo/examples"))
     507        self.assertEquals(linkrel, "../howto/")
     508
     509
     510    def test_getLinkrelToUncle(self):
     511        """
     512        If the doc directory is a sibling of the parent of the resource
     513        directory, the linkrel should make use of multiple '..'s and a named
     514        segment.
     515        """
     516        linkrel = self.builder.getLinkrel(FilePath("/foo/howto"),
     517                                          FilePath("/foo/examples/quotes"))
     518        self.assertEquals(linkrel, "../../howto/")
     519
     520
     521
     522class ManBuilderTestCase(TestCase, BuilderTestsMixin):
     523    """
     524    Tests for L{ManBuilder}.
     525    """
     526    skip = loreSkip
     527
     528    def setUp(self):
     529        """
     530        Set up a few instance variables that will be useful.
     531
     532        @ivar builder: A plain L{ManBuilder}.
     533        @ivar manDir: A L{FilePath} representing a directory to be used for
     534            containing man pages.
     535        """
     536        BuilderTestsMixin.setUp(self)
     537        self.builder = ManBuilder()
     538        self.manDir = FilePath(self.mktemp())
     539        self.manDir.createDirectory()
     540
     541
     542    def test_noDocumentsFound(self):
     543        """
     544        L{ManBuilder.build} raises L{NoDocumentsFound} if there are no
     545        .1 files in the given directory.
     546        """
     547        self.assertRaises(NoDocumentsFound, self.builder.build, self.manDir)
     548
     549
     550    def test_build(self):
     551        """
     552        Check that L{ManBuilder.build} find the man page in the directory, and
     553        successfully produce a Lore content.
     554        """
     555        manContent = self.getArbitraryManInput()
     556        self.manDir.child('test1.1').setContent(manContent)
     557        self.builder.build(self.manDir)
     558        output = self.manDir.child('test1-man.xhtml').getContent()
     559        expected = self.getArbitraryManLoreOutput()
     560        # No-op on *nix, fix for windows
     561        expected = expected.replace('\n', os.linesep)
     562        self.assertEquals(output, expected)
     563
     564
     565    def test_toHTML(self):
     566        """
     567        Check that the content output by C{build} is compatible as input of
     568        L{DocBuilder.build}.
     569        """
     570        manContent = self.getArbitraryManInput()
     571        self.manDir.child('test1.1').setContent(manContent)
     572        self.builder.build(self.manDir)
     573
     574        templateFile = self.manDir.child("template.tpl")
     575        templateFile.setContent(DocBuilderTestCase.template)
     576        docBuilder = DocBuilder()
     577        docBuilder.build("1.2.3", self.manDir, self.manDir,
     578                         templateFile)
     579        output = self.manDir.child('test1-man.html').getContent()
     580
     581        self.assertXMLEqual(
     582            output,
     583            """\
     584<?xml version="1.0" ?><html>
     585    <head><title>Yo:MANHOLE.1</title></head>
     586    <body>
     587    <div class="content">
     588
     589<span/>
     590
     591<h2>NAME<a name="auto0"/></h2>
     592
     593<p>manhole - Connect to a Twisted Manhole service
     594</p>
     595
     596<h2>SYNOPSIS<a name="auto1"/></h2>
     597
     598<p><strong>manhole</strong> </p>
     599
     600<h2>DESCRIPTION<a name="auto2"/></h2>
     601
     602<p>manhole is a GTK interface to Twisted Manhole services. You can execute python
     603code as if at an interactive Python console inside a running Twisted process
     604with this.</p>
     605
     606</div>
     607    <a href="index.html">Index</a>
     608    <span class="version">Version: 1.2.3</span>
     609    </body>
     610    </html>""")
     611
     612
     613
     614class DistributionBuilderTestBase(BuilderTestsMixin, StructureAssertingMixin,
     615                                   TestCase):
     616    """
     617    Base for tests of L{DistributionBuilder}.
     618    """
     619    skip = loreSkip
     620
     621    def setUp(self):
     622        BuilderTestsMixin.setUp(self)
     623
     624        self.rootDir = FilePath(self.mktemp())
     625        self.rootDir.createDirectory()
     626
     627        self.outputDir = FilePath(self.mktemp())
     628        self.outputDir.createDirectory()
     629        self.builder = DistributionBuilder(self.rootDir, self.outputDir)
     630
     631
     632
     633class DistributionBuilderTest(DistributionBuilderTestBase):
     634
     635    def test_twistedDistribution(self):
     636        """
     637        The Twisted tarball contains everything in the source checkout, with
     638        built documentation.
     639        """
     640        loreInput, loreOutput = self.getArbitraryLoreInputAndOutput("10.0.0")
     641        manInput1 = self.getArbitraryManInput()
     642        manOutput1 = self.getArbitraryManHTMLOutput("10.0.0", "../howto/")
     643        manInput2 = self.getArbitraryManInput()
     644        manOutput2 = self.getArbitraryManHTMLOutput("10.0.0", "../howto/")
     645        coreIndexInput, coreIndexOutput = self.getArbitraryLoreInputAndOutput(
     646            "10.0.0", prefix="howto/")
     647
     648        structure = {
     649            "README": "Twisted",
     650            "unrelated": "x",
     651            "LICENSE": "copyright!",
     652            "setup.py": "import toplevel",
     653            "bin": {"web": {"websetroot": "SET ROOT"},
     654                    "twistd": "TWISTD"},
     655            "twisted":
     656                {"web":
     657                     {"__init__.py": "import WEB",
     658                      "topfiles": {"setup.py": "import WEBINSTALL",
     659                                   "README": "WEB!"}},
     660                 "words": {"__init__.py": "import WORDS"},
     661                 "plugins": {"twisted_web.py": "import WEBPLUG",
     662                             "twisted_words.py": "import WORDPLUG"}},
     663            "doc": {"web": {"howto": {"index.xhtml": loreInput},
     664                            "man": {"websetroot.1": manInput2}},
     665                    "core": {"howto": {"template.tpl": self.template},
     666                             "man": {"twistd.1": manInput1},
     667                             "index.xhtml": coreIndexInput}}}
     668
     669        outStructure = {
     670            "README": "Twisted",
     671            "unrelated": "x",
     672            "LICENSE": "copyright!",
     673            "setup.py": "import toplevel",
     674            "bin": {"web": {"websetroot": "SET ROOT"},
     675                    "twistd": "TWISTD"},
     676            "twisted":
     677                {"web": {"__init__.py": "import WEB",
     678                         "topfiles": {"setup.py": "import WEBINSTALL",
     679                                      "README": "WEB!"}},
     680                 "words": {"__init__.py": "import WORDS"},
     681                 "plugins": {"twisted_web.py": "import WEBPLUG",
     682                             "twisted_words.py": "import WORDPLUG"}},
     683            "doc": {"web": {"howto": {"index.html": loreOutput},
     684                            "man": {"websetroot.1": manInput2,
     685                                    "websetroot-man.html": manOutput2}},
     686                    "core": {"howto": {"template.tpl": self.template},
     687                             "man": {"twistd.1": manInput1,
     688                                     "twistd-man.html": manOutput1},
     689                             "index.html": coreIndexOutput}}}
     690
     691        self.createStructure(self.rootDir, structure)
     692
     693        outputFile = self.builder.buildTwisted("10.0.0")
     694
     695        self.assertExtractedStructure(outputFile, outStructure)
     696
     697
     698    def test_stageFileStagesFiles(self):
     699        """
     700        L{_stageFile} duplicates the content and metadata of the given file.
     701        """
     702        # Make a test file
     703        inputFile = self.rootDir.child("foo")
     704
     705        # Put some content in it.
     706        inputFile.setContent("bar")
     707
     708        # Put a funny mode on it.
     709        modeReadOnly = stat.S_IRUSR|stat.S_IRGRP|stat.S_IROTH
     710        inputFile.chmod(modeReadOnly)
     711
     712        # Test the first: stage the file into an existing directory.
     713        outputFile1 = self.outputDir.child("foo")
     714
     715        # Test the second: stage the file into a new directory.
     716        outputFile2 = self.outputDir.preauthChild("sub/dir/foo")
     717
     718        for outputFile in [outputFile1, outputFile2]:
     719            _stageFile(inputFile, outputFile)
     720
     721            # Check the contents of the staged file
     722            self.failUnlessEqual(outputFile.open("r").read(), "bar")
     723
     724            # Check the mode of the staged file
     725            outputFile.restat()
     726            self.assertEqual(outputFile.statinfo.st_mode,
     727                    (modeReadOnly | stat.S_IFREG))
     728
     729
     730    def test_stageFileWillNotOverwrite(self):
     731        """
     732        L{_stageFile} raises an exception if asked to overwrite an output file.
     733        """
     734        # Make a test file
     735        inputFile = self.rootDir.child("foo")
     736        inputFile.setContent("bar")
     737
     738        # Make an output file.
     739        outputFile = self.outputDir.child("foo")
     740
     741        # First attempt should work fine.
     742        _stageFile(inputFile, outputFile)
     743
     744        # Second attempt should raise OSError with EEXIST.
     745        exception = self.failUnlessRaises(OSError, _stageFile, inputFile,
     746                outputFile)
     747
     748        self.failUnlessEqual(exception.errno, errno.EEXIST)
     749
     750
     751    def test_stageFileStagesDirectories(self):
     752        """
     753        L{_stageFile} duplicates the content of the given directory.
     754        """
     755        # Make a test directory with stuff in it.
     756        structure = {
     757            "somedirectory": {
     758                "somefile": "some content",
     759                "anotherdirectory": {
     760                    "anotherfile": "other content"}}}
     761        self.createStructure(self.rootDir, structure)
     762        inputDirectory = self.rootDir.child("somedirectory")
     763
     764        # Stage this directory structure
     765        outputDirectory = self.outputDir.child("somedirectory")
     766        _stageFile(inputDirectory, outputDirectory)
     767
     768        # Check that everything was copied across properly.
     769        self.assertStructure(self.outputDir, structure)
     770
     771
     772    def test_stageFileFiltersBytecode(self):
     773        """
     774        L{_stageFile} ignores Python bytecode files.
     775        """
     776        # Make a test directory with stuff in it.
     777        inputStructure = {
     778            "somepackage": {
     779                "__init__.py": "",
     780                "__init__.pyc": "gibberish",
     781                "__init__.pyo": "more gibberish",
     782                "module.py": "import this",
     783                "module.pyc": "extra gibberish",
     784                "module.pyo": "bonus gibberish",
     785                "datafile.xqz": "A file with an unknown extension"},
     786            "somemodule.py": "import that",
     787            "somemodule.pyc": "surplus gibberish",
     788            "somemodule.pyo": "sundry gibberish"}
     789        self.createStructure(self.rootDir, inputStructure)
     790
     791        # Stage this directory structure
     792        for child in self.rootDir.children():
     793            dest = self.outputDir.child(child.basename())
     794            _stageFile(child, dest)
     795
     796        # Check that everything but bytecode files has been copied across.
     797        outputStructure = {
     798            "somepackage": {
     799                # Ordinary Python files should be copied.
     800                "__init__.py": "",
     801                "module.py": "import this",
     802
     803                # .pyc and .pyc files should be ignored.
     804
     805                # Other unknown files should be copied too.
     806                "datafile.xqz": "A file with an unknown extension"},
     807            # Individually staged files should be copied, unless they're
     808            # bytecode files.
     809            "somemodule.py": "import that"}
     810        self.assertStructure(self.outputDir, outputStructure)
     811
     812
     813    def test_stageFileFiltersVCSMetadata(self):
     814        """
     815        L{_stageFile} ignores common VCS directories.
     816        """
     817        # Make a test directory with stuff in it.
     818        inputStructure = {
     819            # Twisted's official repository is Subversion.
     820            ".svn": {
     821                "svn-data": "some Subversion data"},
     822            # Twisted has a semi-official bzr mirror of the svn repository.
     823            ".bzr": {
     824                "bzr-data": "some Bazaar data"},
     825            # git-svn is a popular way for git users to deal with svn
     826            # repositories.
     827            ".git": {
     828                "git-data": "some Git data"},
     829            "somepackage": {
     830                # Subversion litters its .svn directories everywhere, not just
     831                # the top-level.
     832                ".svn": {
     833                    "svn-data": "more Subversion data"},
     834                "__init__.py": "",
     835                "module.py": "import this"},
     836            "somemodule.py": "import that"}
     837        self.createStructure(self.rootDir, inputStructure)
     838
     839        # Stage this directory structure
     840        for child in self.rootDir.children():
     841            dest = self.outputDir.child(child.basename())
     842            _stageFile(child, dest)
     843
     844        # Check that everything but VCS files has been copied across.
     845        outputStructure = {
     846            # No VCS files in the root.
     847            "somepackage": {
     848                # Ordinary Python files should be copied.
     849                "__init__.py": "",
     850                "module.py": "import this",
     851
     852                # No VCS files in the package, either.
     853                },
     854
     855            # Individually staged files should be copied, unless they're
     856            # bytecode files.
     857            "somemodule.py": "import that"}
     858        self.assertStructure(self.outputDir, outputStructure)
     859
     860
     861    def test_stageFileHandlesEXDEV(self):
     862        """
     863        L{_stageFile} should fall back to copying if os.link raises EXDEV.
     864        """
     865        def mock_link(src, dst):
     866            raise OSError(errno.EXDEV, "dummy error")
     867
     868        # Mock out os.link so that it always fails with EXDEV.
     869        self.patch(os, "link", mock_link)
     870
     871        # Staging a file should still work properly.
     872
     873        # Make a test file
     874        inputFile = self.rootDir.child("foo")
     875        inputFile.setContent("bar")
     876        modeReadOnly = stat.S_IRUSR|stat.S_IRGRP|stat.S_IROTH
     877        inputFile.chmod(modeReadOnly)
     878
     879        # Stage the file into an existing directory.
     880        outputFile = self.outputDir.child("foo")
     881        _stageFile(inputFile, outputFile)
     882
     883        # Check the contents of the staged file
     884        self.failUnlessEqual(outputFile.open("r").read(), "bar")
     885
     886        # Check the mode of the staged file
     887        outputFile.restat()
     888        self.assertEqual(outputFile.statinfo.st_mode,
     889                (modeReadOnly | stat.S_IFREG))
     890
     891    if not getattr(os, "link", None):
     892        test_stageFileHandlesEXDEV.skip = "OS does not support hard-links"
     893
     894
     895    def test_twistedDistributionExcludesWeb2AndVFSAndAdmin(self):
     896        """
     897        The main Twisted distribution does not include web2 or vfs, or the
     898        bin/admin directory.
     899        """
     900        loreInput, loreOutput = self.getArbitraryLoreInputAndOutput("10.0.0")
     901        coreIndexInput, coreIndexOutput = self.getArbitraryLoreInputAndOutput(
     902            "10.0.0", prefix="howto/")
     903
     904        structure = {
     905            "README": "Twisted",
     906            "unrelated": "x",
     907            "LICENSE": "copyright!",
     908            "setup.py": "import toplevel",
     909            "bin": {"web2": {"websetroot": "SET ROOT"},
     910                    "vfs": {"vfsitup": "hee hee"},
     911                    "twistd": "TWISTD",
     912                    "admin": {"build-a-thing": "yay"}},
     913            "twisted":
     914                {"web2":
     915                     {"__init__.py": "import WEB",
     916                      "topfiles": {"setup.py": "import WEBINSTALL",
     917                                   "README": "WEB!"}},
     918                 "vfs":
     919                     {"__init__.py": "import VFS",
     920                      "blah blah": "blah blah"},
     921                 "words": {"__init__.py": "import WORDS"},
     922                 "plugins": {"twisted_web.py": "import WEBPLUG",
     923                             "twisted_words.py": "import WORDPLUG",
     924                             "twisted_web2.py": "import WEB2",
     925                             "twisted_vfs.py": "import VFS"}},
     926            "doc": {"web2": {"excluded!": "yay"},
     927                    "vfs": {"unrelated": "whatever"},
     928                    "core": {"howto": {"template.tpl": self.template},
     929                             "index.xhtml": coreIndexInput}}}
     930
     931        outStructure = {
     932            "README": "Twisted",
     933            "unrelated": "x",
     934            "LICENSE": "copyright!",
     935            "setup.py": "import toplevel",
     936            "bin": {"twistd": "TWISTD"},
     937            "twisted":
     938                {"words": {"__init__.py": "import WORDS"},
     939                 "plugins": {"twisted_web.py": "import WEBPLUG",
     940                             "twisted_words.py": "import WORDPLUG"}},
     941            "doc": {"core": {"howto": {"template.tpl": self.template},
     942                             "index.html": coreIndexOutput}}}
     943        self.createStructure(self.rootDir, structure)
     944
     945        outputFile = self.builder.buildTwisted("10.0.0")
     946
     947        self.assertExtractedStructure(outputFile, outStructure)
     948
     949
     950    def test_subProjectLayout(self):
     951        """
     952        The subproject tarball includes files like so:
     953
     954        1. twisted/<subproject>/topfiles defines the files that will be in the
     955           top level in the tarball, except LICENSE, which comes from the real
     956           top-level directory.
     957        2. twisted/<subproject> is included, but without the topfiles entry
     958           in that directory. No other twisted subpackages are included.
     959        3. twisted/plugins/twisted_<subproject>.py is included, but nothing
     960           else in plugins is.
     961        """
     962        structure = {
     963            "README": "HI!@",
     964            "unrelated": "x",
     965            "LICENSE": "copyright!",
     966            "setup.py": "import toplevel",
     967            "bin": {"web": {"websetroot": "SET ROOT"},
     968                    "words": {"im": "#!im"}},
     969            "twisted":
     970                {"web":
     971                     {"__init__.py": "import WEB",
     972                      "topfiles": {"setup.py": "import WEBINSTALL",
     973                                   "README": "WEB!"}},
     974                 "words": {"__init__.py": "import WORDS"},
     975                 "plugins": {"twisted_web.py": "import WEBPLUG",
     976                             "twisted_words.py": "import WORDPLUG"}}}
     977
     978        outStructure = {
     979            "README": "WEB!",
     980            "LICENSE": "copyright!",
     981            "setup.py": "import WEBINSTALL",
     982            "bin": {"websetroot": "SET ROOT"},
     983            "twisted": {"web": {"__init__.py": "import WEB"},
     984                        "plugins": {"twisted_web.py": "import WEBPLUG"}}}
     985
     986        self.createStructure(self.rootDir, structure)
     987
     988        outputFile = self.builder.buildSubProject("web", "0.3.0")
     989
     990        self.assertExtractedStructure(outputFile, outStructure)
     991
     992
     993    def test_minimalSubProjectLayout(self):
     994        """
     995        L{DistributionBuilder.buildSubProject} works with minimal subprojects.
     996        """
     997        structure = {
     998            "LICENSE": "copyright!",
     999            "bin": {},
     1000            "twisted":
     1001                {"web": {"__init__.py": "import WEB",
     1002                         "topfiles": {"setup.py": "import WEBINSTALL"}},
     1003                 "plugins": {}}}
     1004
     1005        outStructure = {
     1006            "setup.py": "import WEBINSTALL",
     1007            "LICENSE": "copyright!",
     1008            "twisted": {"web": {"__init__.py": "import WEB"}}}
     1009
     1010        self.createStructure(self.rootDir, structure)
     1011
     1012        outputFile = self.builder.buildSubProject("web", "0.3.0")
     1013
     1014        self.assertExtractedStructure(outputFile, outStructure)
     1015
     1016
     1017    def test_subProjectDocBuilding(self):
     1018        """
     1019        When building a subproject release, documentation should be built with
     1020        lore.
     1021        """
     1022        loreInput, loreOutput = self.getArbitraryLoreInputAndOutput("0.3.0")
     1023        manInput = self.getArbitraryManInput()
     1024        manOutput = self.getArbitraryManHTMLOutput("0.3.0", "../howto/")
     1025        structure = {
     1026            "LICENSE": "copyright!",
     1027            "twisted": {"web": {"__init__.py": "import WEB",
     1028                                "topfiles": {"setup.py": "import WEBINST"}}},
     1029            "doc": {"web": {"howto": {"index.xhtml": loreInput},
     1030                            "man": {"twistd.1": manInput}},
     1031                    "core": {"howto": {"template.tpl": self.template}}
     1032                    }
     1033            }
     1034
     1035        outStructure = {
     1036            "LICENSE": "copyright!",
     1037            "setup.py": "import WEBINST",
     1038            "twisted": {"web": {"__init__.py": "import WEB"}},
     1039            "doc": {"howto": {"index.html": loreOutput},
     1040                    "man": {"twistd.1": manInput,
     1041                            "twistd-man.html": manOutput}}}
     1042
     1043        self.createStructure(self.rootDir, structure)
     1044
     1045        outputFile = self.builder.buildSubProject("web", "0.3.0")
     1046
     1047        self.assertExtractedStructure(outputFile, outStructure)
     1048
     1049
     1050    def test_coreProjectLayout(self):
     1051        """
     1052        The core tarball looks a lot like a subproject tarball, except it
     1053        doesn't include:
     1054
     1055        - Python packages from other subprojects
     1056        - plugins from other subprojects
     1057        - scripts from other subprojects
     1058        """
     1059        indexInput, indexOutput = self.getArbitraryLoreInputAndOutput(
     1060            "8.0.0", prefix="howto/")
     1061        howtoInput, howtoOutput = self.getArbitraryLoreInputAndOutput("8.0.0")
     1062        specInput, specOutput = self.getArbitraryLoreInputAndOutput(
     1063            "8.0.0", prefix="../howto/")
     1064        upgradeInput, upgradeOutput = self.getArbitraryLoreInputAndOutput(
     1065            "8.0.0", prefix="../howto/")
     1066        tutorialInput, tutorialOutput = self.getArbitraryLoreInputAndOutput(
     1067            "8.0.0", prefix="../")
     1068
     1069        structure = {
     1070            "LICENSE": "copyright!",
     1071            "twisted": {"__init__.py": "twisted",
     1072                        "python": {"__init__.py": "python",
     1073                                   "roots.py": "roots!"},
     1074                        "conch": {"__init__.py": "conch",
     1075                                  "unrelated.py": "import conch"},
     1076                        "plugin.py": "plugin",
     1077                        "plugins": {"twisted_web.py": "webplug",
     1078                                    "twisted_whatever.py": "include!",
     1079                                    "cred.py": "include!"},
     1080                        "topfiles": {"setup.py": "import CORE",
     1081                                     "README": "core readme"}},
     1082            "doc": {"core": {"howto": {"template.tpl": self.template,
     1083                                       "index.xhtml": howtoInput,
     1084                                       "tutorial":
     1085                                           {"index.xhtml": tutorialInput}},
     1086                             "specifications": {"index.xhtml": specInput},
     1087                             "upgrades": {"index.xhtml": upgradeInput},
     1088                             "examples": {"foo.py": "foo.py"},
     1089                             "index.xhtml": indexInput},
     1090                    "web": {"howto": {"index.xhtml": "webindex"}}},
     1091            "bin": {"twistd": "TWISTD",
     1092                    "web": {"websetroot": "websetroot"}}
     1093            }
     1094
     1095        outStructure = {
     1096            "LICENSE": "copyright!",
     1097            "setup.py": "import CORE",
     1098            "README": "core readme",
     1099            "twisted": {"__init__.py": "twisted",
     1100                        "python": {"__init__.py": "python",
     1101                                   "roots.py": "roots!"},
     1102                        "plugin.py": "plugin",
     1103                        "plugins": {"twisted_whatever.py": "include!",
     1104                                    "cred.py": "include!"}},
     1105            "doc": {"howto": {"template.tpl": self.template,
     1106                              "index.html": howtoOutput,
     1107                              "tutorial": {"index.html": tutorialOutput}},
     1108                    "specifications": {"index.html": specOutput},
     1109                    "upgrades": {"index.html": upgradeOutput},
     1110                    "examples": {"foo.py": "foo.py"},
     1111                    "index.html": indexOutput},
     1112            "bin": {"twistd": "TWISTD"},
     1113            }
     1114
     1115        self.createStructure(self.rootDir, structure)
     1116        outputFile = self.builder.buildCore("8.0.0")
     1117        self.assertExtractedStructure(outputFile, outStructure)
     1118
     1119
     1120    def test_apiBaseURL(self):
     1121        """
     1122        L{DistributionBuilder} builds documentation with the specified
     1123        API base URL.
     1124        """
     1125        apiBaseURL = "http://%s"
     1126        builder = DistributionBuilder(self.rootDir, self.outputDir,
     1127                                      apiBaseURL=apiBaseURL)
     1128        loreInput, loreOutput = self.getArbitraryLoreInputAndOutput(
     1129            "0.3.0", apiBaseURL=apiBaseURL)
     1130        structure = {
     1131            "LICENSE": "copyright!",
     1132            "twisted": {"web": {"__init__.py": "import WEB",
     1133                                "topfiles": {"setup.py": "import WEBINST"}}},
     1134            "doc": {"web": {"howto": {"index.xhtml": loreInput}},
     1135                    "core": {"howto": {"template.tpl": self.template}}
     1136                    }
     1137            }
     1138
     1139        outStructure = {
     1140            "LICENSE": "copyright!",
     1141            "setup.py": "import WEBINST",
     1142            "twisted": {"web": {"__init__.py": "import WEB"}},
     1143            "doc": {"howto": {"index.html": loreOutput}}}
     1144
     1145        self.createStructure(self.rootDir, structure)
     1146        outputFile = builder.buildSubProject("web", "0.3.0")
     1147        self.assertExtractedStructure(outputFile, outStructure)
     1148
     1149
     1150
     1151class IsDistributableTest(TestCase):
     1152    """
     1153    Tests for L{isDistributable}.
     1154    """
     1155
     1156
     1157    def test_fixedNamesExcluded(self):
     1158        """
     1159        L{isDistributable} denies certain fixed names from being packaged.
     1160        """
     1161        self.failUnlessEqual(isDistributable(FilePath("foo/_darcs")), False)
     1162
     1163
     1164    def test_hiddenFilesExcluded(self):
     1165        """
     1166        L{isDistributable} denies names beginning with a ".".
     1167        """
     1168        self.failUnlessEqual(isDistributable(FilePath("foo/.svn")), False)
     1169
     1170
     1171    def test_byteCodeFilesExcluded(self):
     1172        """
     1173        L{isDistributable} denies Python bytecode files.
     1174        """
     1175        self.failUnlessEqual(isDistributable(FilePath("foo/bar.pyc")), False)
     1176        self.failUnlessEqual(isDistributable(FilePath("foo/bar.pyo")), False)
     1177
     1178
     1179    def test_otherFilesIncluded(self):
     1180        """
     1181        L{isDistributable} allows files with other names.
     1182        """
     1183        self.failUnlessEqual(isDistributable(FilePath("foo/bar.py")), True)
     1184        self.failUnlessEqual(isDistributable(FilePath("foo/README")), True)
     1185        self.failUnlessEqual(isDistributable(FilePath("foo/twisted")), True)
     1186
     1187
     1188
     1189class MakeAPIBaseURLTest(TestCase):
     1190    """
     1191    Tests for L{makeAPIBaseURL}.
     1192    """
     1193
     1194
     1195    def test_makeAPIBaseURLIsSubstitutable(self):
     1196        """
     1197        L{makeAPIBaseURL} has a place to subtitute an API name.
     1198        """
     1199        url = makeAPIBaseURL("1.0")
     1200
     1201        # Check that Python's string substitution actually works on this
     1202        # string.
     1203        url % ("foo",)
     1204
     1205
     1206
     1207class FilePathDeltaTest(TestCase):
     1208    """
     1209    Tests for L{filePathDelta}.
     1210    """
     1211
     1212    def test_filePathDeltaSubdir(self):
     1213        """
     1214        L{filePathDelta} can create a simple relative path to a child path.
     1215        """
     1216        self.assertEquals(filePathDelta(FilePath("/foo/bar"),
     1217                                        FilePath("/foo/bar/baz")),
     1218                          ["baz"])
     1219
     1220
     1221    def test_filePathDeltaSiblingDir(self):
     1222        """
     1223        L{filePathDelta} can traverse upwards to create relative paths to
     1224        siblings.
     1225        """
     1226        self.assertEquals(filePathDelta(FilePath("/foo/bar"),
     1227                                        FilePath("/foo/baz")),
     1228                          ["..", "baz"])
     1229
     1230
     1231    def test_filePathNoCommonElements(self):
     1232        """
     1233        L{filePathDelta} can create relative paths to totally unrelated paths
     1234        for maximum portability.
     1235        """
     1236        self.assertEquals(filePathDelta(FilePath("/foo/bar"),
     1237                                        FilePath("/baz/quux")),
     1238                          ["..", "..", "baz", "quux"])
     1239
     1240
     1241
  • twisted/python/test/test_dist.py

    diff --git a/twisted/python/test/test_dist.py b/twisted/python/test/test_dist.py
    index 68571eb..6d887e4 100644
    a b class GetDataFilesTests(TestCase): 
    285285        subDir.child("bar.txt").touch()
    286286
    287287        # Subdirectory contains another VCS dir, with more ignorable data.
    288         subVcsDir = subDir.child("CVS")
     288        subVcsDir = subDir.child("_darcs")
    289289        subVcsDir.makedirs()
    290290        subVcsDir.child("data.txt").touch()
    291291
  • twisted/python/test/test_release.py

    diff --git a/twisted/python/test/test_release.py b/twisted/python/test/test_release.py
    index 03484ca..d62c04f 100644
    a b only ever performed on Linux. 
    1111
    1212import warnings
    1313import operator
    14 import os, sys, signal, stat, errno
     14import os, sys, signal
    1515from StringIO import StringIO
    16 import tarfile
    17 from xml.dom import minidom as dom
    1816
    1917from datetime import date
    2018
    from twisted.python._release import replaceProjectVersion 
    3230from twisted.python._release import updateTwistedVersionInformation, Project
    3331from twisted.python._release import generateVersionFileData
    3432from twisted.python._release import changeAllProjectVersions
    35 from twisted.python._release import VERSION_OFFSET, DocBuilder, ManBuilder
    36 from twisted.python._release import NoDocumentsFound, filePathDelta
     33from twisted.python._release import VERSION_OFFSET
    3734from twisted.python._release import CommandFailed, BookBuilder
    38 from twisted.python._release import DistributionBuilder, APIBuilder
     35from twisted.python._release import APIBuilder
    3936from twisted.python._release import BuildAPIDocsScript
    4037from twisted.python._release import buildAllTarballs, runCommand
    4138from twisted.python._release import UncleanWorkingDirectory, NotWorkingDirectory
    4239from twisted.python._release import ChangeVersionsScript, BuildTarballsScript
    4340from twisted.python._release import NewsBuilder
    44 from twisted.python._release import makeAPIBaseURL, _stageFile, isDistributable
     41
     42from twisted.python.test.test__dist import loreSkip, StructureAssertingMixin
     43from twisted.python.test.test__dist import BuilderTestsMixin
     44from twisted.python.test.test__dist import DistributionBuilderTestBase
    4545
    4646if sys.platform != 'linux2':
    4747    skip = "Release toolchain only supported on Linux."
    else: 
    4949    skip = None
    5050
    5151
    52 # Check a bunch of dependencies to skip tests if necessary.
    53 try:
    54     from twisted.lore.scripts import lore
    55 except ImportError:
    56     loreSkip = "Lore is not present."
    57 else:
    58     loreSkip = skip
    59 
    60 
    6152try:
    6253    from popen2 import Popen4
    6354except ImportError:
    else: 
    9182    svnSkip = "svn or svnadmin is not present."
    9283
    9384
    94 # When we test that scripts are installed with the "correct" permissions, we
    95 # expect the "correct" permissions to be rwxr-xr-x
    96 SCRIPT_PERMS = (
    97         stat.S_IRWXU # rwx
    98         | stat.S_IRGRP | stat.S_IXGRP # r-x
    99         | stat.S_IROTH | stat.S_IXOTH) # r-x
    100 
    101 
    10285def genVersion(*args, **kwargs):
    10386    """
    10487    A convenience for generating _version.py data.
    def genVersion(*args, **kwargs): 
    11093
    11194
    11295
    113 class StructureAssertingMixin(object):
    114     """
    115     A mixin for L{TestCase} subclasses which provides some methods for asserting
    116     the structure and contents of directories and files on the filesystem.
    117     """
    118     def createStructure(self, parent, dirDict, origRoot=None):
    119         """
    120         Create a set of directories and files given a dict defining their
    121         structure.
    122 
    123         @param parent: The directory in which to create the structure.  It must
    124             already exist.
    125         @type parent: L{FilePath}
    126 
    127         @param dirDict: The dict defining the structure. Keys should be strings
    128             naming files, values should be strings describing file contents OR
    129             dicts describing subdirectories.  All files are written in binary
    130             mode.  Any string values are assumed to describe text files and
    131             will have their newlines replaced with the platform-native newline
    132             convention.  For example::
    133 
    134                 {"foofile": "foocontents",
    135                  "bardir": {"barfile": "bar\ncontents"}}
    136         @type dirDict: C{dict}
    137 
    138         @param origRoot: The directory provided as C{parent} in the original
    139             invocation of C{createStructure}. Leave this as C{None}, it's used
    140             in recursion.
    141         @type origRoot: L{FilePath} or C{None}
    142         """
    143         if origRoot is None:
    144             origRoot = parent
    145 
    146         for x in dirDict:
    147             child = parent.child(x)
    148             if isinstance(dirDict[x], dict):
    149                 child.createDirectory()
    150                 self.createStructure(child, dirDict[x], origRoot=origRoot)
    151 
    152                 # If x is in a bin directory, make sure children
    153                 # representing files have the executable bit set.
    154                 if "bin" in child.segmentsFrom(origRoot):
    155                     for script in [k for (k,v) in dirDict[x].items()
    156                             if isinstance(v, basestring)]:
    157                         scriptPath = child.child(script)
    158                         scriptPath.chmod(SCRIPT_PERMS)
    159 
    160             else:
    161                 child.setContent(dirDict[x].replace('\n', os.linesep))
    162 
    163     def assertStructure(self, root, dirDict):
    164         """
    165         Assert that a directory is equivalent to one described by a dict.
    166 
    167         @param root: The filesystem directory to compare.
    168         @type root: L{FilePath}
    169         @param dirDict: The dict that should describe the contents of the
    170             directory. It should be the same structure as the C{dirDict}
    171             parameter to L{createStructure}.
    172         @type dirDict: C{dict}
    173         """
    174         children = [x.basename() for x in root.children()]
    175         for x in dirDict:
    176             child = root.child(x)
    177             if isinstance(dirDict[x], dict):
    178                 self.assertTrue(child.isdir(), "%s is not a dir!"
    179                                 % (child.path,))
    180                 self.assertStructure(child, dirDict[x])
    181 
    182                 # If x is in a bin directory, make sure children
    183                 # representing files have the executable bit set.
    184                 if "/bin" in child.path:
    185                     for script in [k for (k,v) in dirDict[x].items()
    186                             if isinstance(v, basestring)]:
    187                         scriptPath = child.child(script)
    188                         scriptPath.restat()
    189                         # What with SVN and umask and all that jazz, all we can
    190                         # really check is that these scripts have at least one
    191                         # executable bit set.
    192                         self.assertTrue(scriptPath.statinfo.st_mode &
    193                                 (stat.S_IXUSR|stat.S_IXGRP|stat.S_IXOTH),
    194                                 "File %r should be executable"
    195                                 % (scriptPath.path,))
    196             else:
    197                 a = child.getContent().replace(os.linesep, '\n')
    198                 self.assertEquals(a, dirDict[x], child.path)
    199             children.remove(x)
    200         if children:
    201             self.fail("There were extra children in %s: %s"
    202                       % (root.path, children))
    203 
    204 
    205     def assertExtractedStructure(self, outputFile, dirDict):
    206         """
    207         Assert that a tarfile content is equivalent to one described by a dict.
    208 
    209         @param outputFile: The tar file built by L{DistributionBuilder}.
    210         @type outputFile: L{FilePath}.
    211         @param dirDict: The dict that should describe the contents of the
    212             directory. It should be the same structure as the C{dirDict}
    213             parameter to L{createStructure}.
    214         @type dirDict: C{dict}
    215         """
    216         tarFile = tarfile.TarFile.open(outputFile.path, "r:bz2")
    217         extracted = FilePath(self.mktemp())
    218         extracted.createDirectory()
    219         for info in tarFile:
    220             tarFile.extract(info, path=extracted.path)
    221         self.assertStructure(extracted.children()[0], dirDict)
    222 
    223 
    224 
    22596class ChangeVersionTest(TestCase, StructureAssertingMixin):
    22697    """
    22798    Twisted has the ability to change versions.
    class VersionWritingTest(TestCase): 
    530401
    531402
    532403
    533 class BuilderTestsMixin(object):
    534     """
    535     A mixin class which provides various methods for creating sample Lore input
    536     and output.
    537 
    538     @cvar template: The lore template that will be used to prepare sample
    539     output.
    540     @type template: C{str}
    541 
    542     @ivar docCounter: A counter which is incremented every time input is
    543         generated and which is included in the documents.
    544     @type docCounter: C{int}
    545     """
    546     template = '''
    547     <html>
    548     <head><title>Yo:</title></head>
    549     <body>
    550     <div class="body" />
    551     <a href="index.html">Index</a>
    552     <span class="version">Version: </span>
    553     </body>
    554     </html>
    555     '''
    556 
    557     def setUp(self):
    558         """
    559         Initialize the doc counter which ensures documents are unique.
    560         """
    561         self.docCounter = 0
    562 
    563 
    564     def assertXMLEqual(self, first, second):
    565         """
    566         Verify that two strings represent the same XML document.
    567         """
    568         self.assertEqual(
    569             dom.parseString(first).toxml(),
    570             dom.parseString(second).toxml())
    571 
    572 
    573     def getArbitraryOutput(self, version, counter, prefix="", apiBaseURL="%s"):
    574         """
    575         Get the correct HTML output for the arbitrary input returned by
    576         L{getArbitraryLoreInput} for the given parameters.
    577 
    578         @param version: The version string to include in the output.
    579         @type version: C{str}
    580         @param counter: A counter to include in the output.
    581         @type counter: C{int}
    582         """
    583         document = """\
    584 <?xml version="1.0"?><html>
    585     <head><title>Yo:Hi! Title: %(count)d</title></head>
    586     <body>
    587     <div class="content">Hi! %(count)d<div class="API"><a href="%(foobarLink)s" title="foobar">foobar</a></div></div>
    588     <a href="%(prefix)sindex.html">Index</a>
    589     <span class="version">Version: %(version)s</span>
    590     </body>
    591     </html>"""
    592         # Try to normalize irrelevant whitespace.
    593         return dom.parseString(
    594             document % {"count": counter, "prefix": prefix,
    595                         "version": version,
    596                         "foobarLink": apiBaseURL % ("foobar",)}).toxml('utf-8')
    597 
    598 
    599     def getArbitraryLoreInput(self, counter):
    600         """
    601         Get an arbitrary, unique (for this test case) string of lore input.
    602 
    603         @param counter: A counter to include in the input.
    604         @type counter: C{int}
    605         """
    606         template = (
    607             '<html>'
    608             '<head><title>Hi! Title: %(count)s</title></head>'
    609             '<body>'
    610             'Hi! %(count)s'
    611             '<div class="API">foobar</div>'
    612             '</body>'
    613             '</html>')
    614         return template % {"count": counter}
    615 
    616 
    617     def getArbitraryLoreInputAndOutput(self, version, prefix="",
    618                                        apiBaseURL="%s"):
    619         """
    620         Get an input document along with expected output for lore run on that
    621         output document, assuming an appropriately-specified C{self.template}.
    622 
    623         @param version: A version string to include in the input and output.
    624         @type version: C{str}
    625         @param prefix: The prefix to include in the link to the index.
    626         @type prefix: C{str}
    627 
    628         @return: A two-tuple of input and expected output.
    629         @rtype: C{(str, str)}.
    630         """
    631         self.docCounter += 1
    632         return (self.getArbitraryLoreInput(self.docCounter),
    633                 self.getArbitraryOutput(version, self.docCounter,
    634                                         prefix=prefix, apiBaseURL=apiBaseURL))
    635 
    636 
    637     def getArbitraryManInput(self):
    638         """
    639         Get an arbitrary man page content.
    640         """
    641         return """.TH MANHOLE "1" "August 2001" "" ""
    642 .SH NAME
    643 manhole \- Connect to a Twisted Manhole service
    644 .SH SYNOPSIS
    645 .B manhole
    646 .SH DESCRIPTION
    647 manhole is a GTK interface to Twisted Manhole services. You can execute python
    648 code as if at an interactive Python console inside a running Twisted process
    649 with this."""
    650 
    651 
    652     def getArbitraryManLoreOutput(self):
    653         """
    654         Get an arbitrary lore input document which represents man-to-lore
    655         output based on the man page returned from L{getArbitraryManInput}
    656         """
    657         return """\
    658 <?xml version="1.0"?>
    659 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
    660     "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
    661 <html><head>
    662 <title>MANHOLE.1</title></head>
    663 <body>
    664 
    665 <h1>MANHOLE.1</h1>
    666 
    667 <h2>NAME</h2>
    668 
    669 <p>manhole - Connect to a Twisted Manhole service
    670 </p>
    671 
    672 <h2>SYNOPSIS</h2>
    673 
    674 <p><strong>manhole</strong> </p>
    675 
    676 <h2>DESCRIPTION</h2>
    677 
    678 <p>manhole is a GTK interface to Twisted Manhole services. You can execute python
    679 code as if at an interactive Python console inside a running Twisted process
    680 with this.</p>
    681 
    682 </body>
    683 </html>
    684 """
    685 
    686     def getArbitraryManHTMLOutput(self, version, prefix=""):
    687         """
    688         Get an arbitrary lore output document which represents the lore HTML
    689         output based on the input document returned from
    690         L{getArbitraryManLoreOutput}.
    691 
    692         @param version: A version string to include in the document.
    693         @type version: C{str}
    694         @param prefix: The prefix to include in the link to the index.
    695         @type prefix: C{str}
    696         """
    697         # Try to normalize the XML a little bit.
    698         return dom.parseString("""\
    699 <?xml version="1.0" ?><html>
    700     <head><title>Yo:MANHOLE.1</title></head>
    701     <body>
    702     <div class="content">
    703 
    704 <span/>
    705 
    706 <h2>NAME<a name="auto0"/></h2>
    707 
    708 <p>manhole - Connect to a Twisted Manhole service
    709 </p>
    710 
    711 <h2>SYNOPSIS<a name="auto1"/></h2>
    712 
    713 <p><strong>manhole</strong> </p>
    714 
    715 <h2>DESCRIPTION<a name="auto2"/></h2>
    716 
    717 <p>manhole is a GTK interface to Twisted Manhole services. You can execute python
    718 code as if at an interactive Python console inside a running Twisted process
    719 with this.</p>
    720 
    721 </div>
    722     <a href="%(prefix)sindex.html">Index</a>
    723     <span class="version">Version: %(version)s</span>
    724     </body>
    725     </html>""" % {
    726             'prefix': prefix, 'version': version}).toxml("utf-8")
    727 
    728 
    729 
    730 class DocBuilderTestCase(TestCase, BuilderTestsMixin):
    731     """
    732     Tests for L{DocBuilder}.
    733 
    734     Note for future maintainers: The exact byte equality assertions throughout
    735     this suite may need to be updated due to minor differences in lore. They
    736     should not be taken to mean that Lore must maintain the same byte format
    737     forever. Feel free to update the tests when Lore changes, but please be
    738     careful.
    739     """
    740     skip = loreSkip
    741 
    742     def setUp(self):
    743         """
    744         Set up a few instance variables that will be useful.
    745 
    746         @ivar builder: A plain L{DocBuilder}.
    747         @ivar docCounter: An integer to be used as a counter by the
    748             C{getArbitrary...} methods.
    749         @ivar howtoDir: A L{FilePath} representing a directory to be used for
    750             containing Lore documents.
    751         @ivar templateFile: A L{FilePath} representing a file with
    752             C{self.template} as its content.
    753         """
    754         BuilderTestsMixin.setUp(self)
    755         self.builder = DocBuilder()
    756         self.howtoDir = FilePath(self.mktemp())
    757         self.howtoDir.createDirectory()
    758         self.templateFile = self.howtoDir.child("template.tpl")
    759         self.templateFile.setContent(self.template)
    760 
    761 
    762     def test_build(self):
    763         """
    764         The L{DocBuilder} runs lore on all .xhtml files within a directory.
    765         """
    766         version = "1.2.3"
    767         input1, output1 = self.getArbitraryLoreInputAndOutput(version)
    768         input2, output2 = self.getArbitraryLoreInputAndOutput(version)
    769 
    770         self.howtoDir.child("one.xhtml").setContent(input1)
    771         self.howtoDir.child("two.xhtml").setContent(input2)
    772 
    773         self.builder.build(version, self.howtoDir, self.howtoDir,
    774                            self.templateFile)
    775         out1 = self.howtoDir.child('one.html')
    776         out2 = self.howtoDir.child('two.html')
    777         self.assertXMLEqual(out1.getContent(), output1)
    778         self.assertXMLEqual(out2.getContent(), output2)
    779 
    780 
    781     def test_noDocumentsFound(self):
    782         """
    783         The C{build} method raises L{NoDocumentsFound} if there are no
    784         .xhtml files in the given directory.
    785         """
    786         self.assertRaises(
    787             NoDocumentsFound,
    788             self.builder.build, "1.2.3", self.howtoDir, self.howtoDir,
    789             self.templateFile)
    790 
    791 
    792     def test_parentDocumentLinking(self):
    793         """
    794         The L{DocBuilder} generates correct links from documents to
    795         template-generated links like stylesheets and index backreferences.
    796         """
    797         input = self.getArbitraryLoreInput(0)
    798         tutoDir = self.howtoDir.child("tutorial")
    799         tutoDir.createDirectory()
    800         tutoDir.child("child.xhtml").setContent(input)
    801         self.builder.build("1.2.3", self.howtoDir, tutoDir, self.templateFile)
    802         outFile = tutoDir.child('child.html')
    803         self.assertIn('<a href="../index.html">Index</a>',
    804                       outFile.getContent())
    805 
    806 
    807     def test_siblingDirectoryDocumentLinking(self):
    808         """
    809         It is necessary to generate documentation in a directory foo/bar where
    810         stylesheet and indexes are located in foo/baz. Such resources should be
    811         appropriately linked to.
    812         """
    813         input = self.getArbitraryLoreInput(0)
    814         resourceDir = self.howtoDir.child("resources")
    815         docDir = self.howtoDir.child("docs")
    816         docDir.createDirectory()
    817         docDir.child("child.xhtml").setContent(input)
    818         self.builder.build("1.2.3", resourceDir, docDir, self.templateFile)
    819         outFile = docDir.child('child.html')
    820         self.assertIn('<a href="../resources/index.html">Index</a>',
    821                       outFile.getContent())
    822 
    823 
    824     def test_apiLinking(self):
    825         """
    826         The L{DocBuilder} generates correct links from documents to API
    827         documentation.
    828         """
    829         version = "1.2.3"
    830         input, output = self.getArbitraryLoreInputAndOutput(version)
    831         self.howtoDir.child("one.xhtml").setContent(input)
    832 
    833         self.builder.build(version, self.howtoDir, self.howtoDir,
    834                            self.templateFile, "scheme:apilinks/%s.ext")
    835         out = self.howtoDir.child('one.html')
    836         self.assertIn(
    837             '<a href="scheme:apilinks/foobar.ext" title="foobar">foobar</a>',
    838             out.getContent())
    839 
    840 
    841     def test_deleteInput(self):
    842         """
    843         L{DocBuilder.build} can be instructed to delete the input files after
    844         generating the output based on them.
    845         """
    846         input1 = self.getArbitraryLoreInput(0)
    847         self.howtoDir.child("one.xhtml").setContent(input1)
    848         self.builder.build("whatever", self.howtoDir, self.howtoDir,
    849                            self.templateFile, deleteInput=True)
    850         self.assertTrue(self.howtoDir.child('one.html').exists())
    851         self.assertFalse(self.howtoDir.child('one.xhtml').exists())
    852 
    853 
    854     def test_doNotDeleteInput(self):
    855         """
    856         Input will not be deleted by default.
    857         """
    858         input1 = self.getArbitraryLoreInput(0)
    859         self.howtoDir.child("one.xhtml").setContent(input1)
    860         self.builder.build("whatever", self.howtoDir, self.howtoDir,
    861                            self.templateFile)
    862         self.assertTrue(self.howtoDir.child('one.html').exists())
    863         self.assertTrue(self.howtoDir.child('one.xhtml').exists())
    864 
    865 
    866     def test_getLinkrelToSameDirectory(self):
    867         """
    868         If the doc and resource directories are the same, the linkrel should be
    869         an empty string.
    870         """
    871         linkrel = self.builder.getLinkrel(FilePath("/foo/bar"),
    872                                           FilePath("/foo/bar"))
    873         self.assertEquals(linkrel, "")
    874 
    875 
    876     def test_getLinkrelToParentDirectory(self):
    877         """
    878         If the doc directory is a child of the resource directory, the linkrel
    879         should make use of '..'.
    880         """
    881         linkrel = self.builder.getLinkrel(FilePath("/foo"),
    882                                           FilePath("/foo/bar"))
    883         self.assertEquals(linkrel, "../")
    884 
    885 
    886     def test_getLinkrelToSibling(self):
    887         """
    888         If the doc directory is a sibling of the resource directory, the
    889         linkrel should make use of '..' and a named segment.
    890         """
    891         linkrel = self.builder.getLinkrel(FilePath("/foo/howto"),
    892                                           FilePath("/foo/examples"))
    893         self.assertEquals(linkrel, "../howto/")
    894 
    895 
    896     def test_getLinkrelToUncle(self):
    897         """
    898         If the doc directory is a sibling of the parent of the resource
    899         directory, the linkrel should make use of multiple '..'s and a named
    900         segment.
    901         """
    902         linkrel = self.builder.getLinkrel(FilePath("/foo/howto"),
    903                                           FilePath("/foo/examples/quotes"))
    904         self.assertEquals(linkrel, "../../howto/")
    905 
    906 
    907 
    908404class APIBuilderTestCase(TestCase):
    909405    """
    910406    Tests for L{APIBuilder}.
    class APIBuilderTestCase(TestCase): 
    1044540
    1045541
    1046542
    1047 class ManBuilderTestCase(TestCase, BuilderTestsMixin):
    1048     """
    1049     Tests for L{ManBuilder}.
    1050     """
    1051     skip = loreSkip
    1052 
    1053     def setUp(self):
    1054         """
    1055         Set up a few instance variables that will be useful.
    1056 
    1057         @ivar builder: A plain L{ManBuilder}.
    1058         @ivar manDir: A L{FilePath} representing a directory to be used for
    1059             containing man pages.
    1060         """
    1061         BuilderTestsMixin.setUp(self)
    1062         self.builder = ManBuilder()
    1063         self.manDir = FilePath(self.mktemp())
    1064         self.manDir.createDirectory()
    1065 
    1066 
    1067     def test_noDocumentsFound(self):
    1068         """
    1069         L{ManBuilder.build} raises L{NoDocumentsFound} if there are no
    1070         .1 files in the given directory.
    1071         """
    1072         self.assertRaises(NoDocumentsFound, self.builder.build, self.manDir)
    1073 
    1074 
    1075     def test_build(self):
    1076         """
    1077         Check that L{ManBuilder.build} find the man page in the directory, and
    1078         successfully produce a Lore content.
    1079         """
    1080         manContent = self.getArbitraryManInput()
    1081         self.manDir.child('test1.1').setContent(manContent)
    1082         self.builder.build(self.manDir)
    1083         output = self.manDir.child('test1-man.xhtml').getContent()
    1084         expected = self.getArbitraryManLoreOutput()
    1085         # No-op on *nix, fix for windows
    1086         expected = expected.replace('\n', os.linesep)
    1087         self.assertEquals(output, expected)
    1088 
    1089 
    1090     def test_toHTML(self):
    1091         """
    1092         Check that the content output by C{build} is compatible as input of
    1093         L{DocBuilder.build}.
    1094         """
    1095         manContent = self.getArbitraryManInput()
    1096         self.manDir.child('test1.1').setContent(manContent)
    1097         self.builder.build(self.manDir)
    1098 
    1099         templateFile = self.manDir.child("template.tpl")
    1100         templateFile.setContent(DocBuilderTestCase.template)
    1101         docBuilder = DocBuilder()
    1102         docBuilder.build("1.2.3", self.manDir, self.manDir,
    1103                          templateFile)
    1104         output = self.manDir.child('test1-man.html').getContent()
    1105 
    1106         self.assertXMLEqual(
    1107             output,
    1108             """\
    1109 <?xml version="1.0" ?><html>
    1110     <head><title>Yo:MANHOLE.1</title></head>
    1111     <body>
    1112     <div class="content">
    1113 
    1114 <span/>
    1115 
    1116 <h2>NAME<a name="auto0"/></h2>
    1117 
    1118 <p>manhole - Connect to a Twisted Manhole service
    1119 </p>
    1120 
    1121 <h2>SYNOPSIS<a name="auto1"/></h2>
    1122 
    1123 <p><strong>manhole</strong> </p>
    1124 
    1125 <h2>DESCRIPTION<a name="auto2"/></h2>
    1126 
    1127 <p>manhole is a GTK interface to Twisted Manhole services. You can execute python
    1128 code as if at an interactive Python console inside a running Twisted process
    1129 with this.</p>
    1130 
    1131 </div>
    1132     <a href="index.html">Index</a>
    1133     <span class="version">Version: 1.2.3</span>
    1134     </body>
    1135     </html>""")
    1136 
    1137 
    1138 
    1139543class BookBuilderTests(TestCase, BuilderTestsMixin):
    1140544    """
    1141545    Tests for L{BookBuilder}.
    class BookBuilderTests(TestCase, BuilderTestsMixin): 
    1456860
    1457861
    1458862
    1459 class FilePathDeltaTest(TestCase):
    1460     """
    1461     Tests for L{filePathDelta}.
    1462     """
    1463 
    1464     def test_filePathDeltaSubdir(self):
    1465         """
    1466         L{filePathDelta} can create a simple relative path to a child path.
    1467         """
    1468         self.assertEquals(filePathDelta(FilePath("/foo/bar"),
    1469                                         FilePath("/foo/bar/baz")),
    1470                           ["baz"])
    1471 
    1472 
    1473     def test_filePathDeltaSiblingDir(self):
    1474         """
    1475         L{filePathDelta} can traverse upwards to create relative paths to
    1476         siblings.
    1477         """
    1478         self.assertEquals(filePathDelta(FilePath("/foo/bar"),
    1479                                         FilePath("/foo/baz")),
    1480                           ["..", "baz"])
    1481 
    1482 
    1483     def test_filePathNoCommonElements(self):
    1484         """
    1485         L{filePathDelta} can create relative paths to totally unrelated paths
    1486         for maximum portability.
    1487         """
    1488         self.assertEquals(filePathDelta(FilePath("/foo/bar"),
    1489                                         FilePath("/baz/quux")),
    1490                           ["..", "..", "baz", "quux"])
    1491 
    1492 
    1493 
    1494863class NewsBuilderTests(TestCase, StructureAssertingMixin):
    1495864    """
    1496865    Tests for L{NewsBuilder}.
    class NewsBuilderTests(TestCase, StructureAssertingMixin): 
    18501219
    18511220
    18521221
    1853 class DistributionBuilderTestBase(BuilderTestsMixin, StructureAssertingMixin,
    1854                                    TestCase):
    1855     """
    1856     Base for tests of L{DistributionBuilder}.
    1857     """
    1858     skip = loreSkip
    1859 
    1860     def setUp(self):
    1861         BuilderTestsMixin.setUp(self)
    1862 
    1863         self.rootDir = FilePath(self.mktemp())
    1864         self.rootDir.createDirectory()
    1865 
    1866         self.outputDir = FilePath(self.mktemp())
    1867         self.outputDir.createDirectory()
    1868         self.builder = DistributionBuilder(self.rootDir, self.outputDir)
    1869 
    1870 
    1871 
    1872 class DistributionBuilderTest(DistributionBuilderTestBase):
    1873 
    1874     def test_twistedDistribution(self):
    1875         """
    1876         The Twisted tarball contains everything in the source checkout, with
    1877         built documentation.
    1878         """
    1879         loreInput, loreOutput = self.getArbitraryLoreInputAndOutput("10.0.0")
    1880         manInput1 = self.getArbitraryManInput()
    1881         manOutput1 = self.getArbitraryManHTMLOutput("10.0.0", "../howto/")
    1882         manInput2 = self.getArbitraryManInput()
    1883         manOutput2 = self.getArbitraryManHTMLOutput("10.0.0", "../howto/")
    1884         coreIndexInput, coreIndexOutput = self.getArbitraryLoreInputAndOutput(
    1885             "10.0.0", prefix="howto/")
    1886 
    1887         structure = {
    1888             "README": "Twisted",
    1889             "unrelated": "x",
    1890             "LICENSE": "copyright!",
    1891             "setup.py": "import toplevel",
    1892             "bin": {"web": {"websetroot": "SET ROOT"},
    1893                     "twistd": "TWISTD"},
    1894             "twisted":
    1895                 {"web":
    1896                      {"__init__.py": "import WEB",
    1897                       "topfiles": {"setup.py": "import WEBINSTALL",
    1898                                    "README": "WEB!"}},
    1899                  "words": {"__init__.py": "import WORDS"},
    1900                  "plugins": {"twisted_web.py": "import WEBPLUG",
    1901                              "twisted_words.py": "import WORDPLUG"}},
    1902             "doc": {"web": {"howto": {"index.xhtml": loreInput},
    1903                             "man": {"websetroot.1": manInput2}},
    1904                     "core": {"howto": {"template.tpl": self.template},
    1905                              "man": {"twistd.1": manInput1},
    1906                              "index.xhtml": coreIndexInput}}}
    1907 
    1908         outStructure = {
    1909             "README": "Twisted",
    1910             "unrelated": "x",
    1911             "LICENSE": "copyright!",
    1912             "setup.py": "import toplevel",
    1913             "bin": {"web": {"websetroot": "SET ROOT"},
    1914                     "twistd": "TWISTD"},
    1915             "twisted":
    1916                 {"web": {"__init__.py": "import WEB",
    1917                          "topfiles": {"setup.py": "import WEBINSTALL",
    1918                                       "README": "WEB!"}},
    1919                  "words": {"__init__.py": "import WORDS"},
    1920                  "plugins": {"twisted_web.py": "import WEBPLUG",
    1921                              "twisted_words.py": "import WORDPLUG"}},
    1922             "doc": {"web": {"howto": {"index.html": loreOutput},
    1923                             "man": {"websetroot.1": manInput2,
    1924                                     "websetroot-man.html": manOutput2}},
    1925                     "core": {"howto": {"template.tpl": self.template},
    1926                              "man": {"twistd.1": manInput1,
    1927                                      "twistd-man.html": manOutput1},
    1928                              "index.html": coreIndexOutput}}}
    1929 
    1930         self.createStructure(self.rootDir, structure)
    1931 
    1932         outputFile = self.builder.buildTwisted("10.0.0")
    1933 
    1934         self.assertExtractedStructure(outputFile, outStructure)
    1935 
    1936 
    1937     def test_stageFileStagesFiles(self):
    1938         """
    1939         _stageFile duplicates the content and metadata of the given file.
    1940         """
    1941         # Make a test file
    1942         inputFile = self.rootDir.child("foo")
    1943 
    1944         # Put some content in it.
    1945         inputFile.setContent("bar")
    1946 
    1947         # Put a funny mode on it.
    1948         modeReadOnly = stat.S_IRUSR|stat.S_IRGRP|stat.S_IROTH
    1949         inputFile.chmod(modeReadOnly)
    1950 
    1951         # Test the first: stage the file into an existing directory.
    1952         outputFile1 = self.outputDir.child("foo")
    1953 
    1954         # Test the second: stage the file into a new directory.
    1955         outputFile2 = self.outputDir.preauthChild("sub/dir/foo")
    1956 
    1957         for outputFile in [outputFile1, outputFile2]:
    1958             _stageFile(inputFile, outputFile)
    1959 
    1960             # Check the contents of the staged file
    1961             self.assertEquals(outputFile.getContent(), "bar")
    1962 
    1963             # Check the mode of the staged file
    1964             outputFile.restat()
    1965             self.assertEquals(
    1966                 outputFile.statinfo.st_mode, (modeReadOnly | stat.S_IFREG))
    1967 
    1968 
    1969     def test_stageFileWillNotOverwrite(self):
    1970         """
    1971         _stageFile raises an exception if asked to overwrite an output file.
    1972         """
    1973         # Make a test file
    1974         inputFile = self.rootDir.child("foo")
    1975         inputFile.setContent("bar")
    1976 
    1977         # Make an output file.
    1978         outputFile = self.outputDir.child("foo")
    1979 
    1980         # First attempt should work fine.
    1981         _stageFile(inputFile, outputFile)
    1982 
    1983         # Second attempt should raise OSError with EEXIST.
    1984         exception = self.assertRaises(
    1985             OSError, _stageFile, inputFile, outputFile)
    1986 
    1987         self.assertEquals(exception.errno, errno.EEXIST)
    1988 
    1989 
    1990     def test_stageFileStagesDirectories(self):
    1991         """
    1992         _stageFile duplicates the content of the given directory.
    1993         """
    1994         # Make a test directory with stuff in it.
    1995         structure = {
    1996             "somedirectory": {
    1997                 "somefile": "some content",
    1998                 "anotherdirectory": {
    1999                     "anotherfile": "other content"}}}
    2000         self.createStructure(self.rootDir, structure)
    2001         inputDirectory = self.rootDir.child("somedirectory")
    2002 
    2003         # Stage this directory structure
    2004         outputDirectory = self.outputDir.child("somedirectory")
    2005         _stageFile(inputDirectory, outputDirectory)
    2006 
    2007         # Check that everything was copied across properly.
    2008         self.assertStructure(self.outputDir, structure)
    2009 
    2010 
    2011     def test_stageFileFiltersBytecode(self):
    2012         """
    2013         _stageFile ignores Python bytecode files.
    2014         """
    2015         # Make a test directory with stuff in it.
    2016         inputStructure = {
    2017             "somepackage": {
    2018                 "__init__.py": "",
    2019                 "__init__.pyc": "gibberish",
    2020                 "__init__.pyo": "more gibberish",
    2021                 "module.py": "import this",
    2022                 "module.pyc": "extra gibberish",
    2023                 "module.pyo": "bonus gibberish",
    2024                 "datafile.xqz": "A file with an unknown extension"},
    2025             "somemodule.py": "import that",
    2026             "somemodule.pyc": "surplus gibberish",
    2027             "somemodule.pyo": "sundry gibberish"}
    2028         self.createStructure(self.rootDir, inputStructure)
    2029 
    2030         # Stage this directory structure
    2031         for child in self.rootDir.children():
    2032             dest = self.outputDir.child(child.basename())
    2033             _stageFile(child, dest)
    2034 
    2035         # Check that everything but bytecode files has been copied across.
    2036         outputStructure = {
    2037             "somepackage": {
    2038                 # Ordinary Python files should be copied.
    2039                 "__init__.py": "",
    2040                 "module.py": "import this",
    2041 
    2042                 # .pyc and .pyc files should be ignored.
    2043 
    2044                 # Other unknown files should be copied too.
    2045                 "datafile.xqz": "A file with an unknown extension"},
    2046             # Individually staged files should be copied, unless they're
    2047             # bytecode files.
    2048             "somemodule.py": "import that"}
    2049         self.assertStructure(self.outputDir, outputStructure)
    2050 
    2051 
    2052     def test_stageFileFiltersVCSMetadata(self):
    2053         """
    2054         _stageFile ignores common VCS directories.
    2055         """
    2056         # Make a test directory with stuff in it.
    2057         inputStructure = {
    2058             # Twisted's official repository is Subversion.
    2059             ".svn": {
    2060                 "svn-data": "some Subversion data"},
    2061             # Twisted has a semi-official bzr mirror of the svn repository.
    2062             ".bzr": {
    2063                 "bzr-data": "some Bazaar data"},
    2064             # git-svn is a popular way for git users to deal with svn
    2065             # repositories.
    2066             ".git": {
    2067                 "git-data": "some Git data"},
    2068             "somepackage": {
    2069                 # Subversion litters its .svn directories everywhere, not just
    2070                 # the top-level.
    2071                 ".svn": {
    2072                     "svn-data": "more Subversion data"},
    2073                 "__init__.py": "",
    2074                 "module.py": "import this"},
    2075             "somemodule.py": "import that"}
    2076         self.createStructure(self.rootDir, inputStructure)
    2077 
    2078         # Stage this directory structure
    2079         for child in self.rootDir.children():
    2080             dest = self.outputDir.child(child.basename())
    2081             _stageFile(child, dest)
    2082 
    2083         # Check that everything but VCS files has been copied across.
    2084         outputStructure = {
    2085             # No VCS files in the root.
    2086             "somepackage": {
    2087                 # Ordinary Python files should be copied.
    2088                 "__init__.py": "",
    2089                 "module.py": "import this",
    2090 
    2091                 # No VCS files in the package, either.
    2092                 },
    2093 
    2094             # Individually staged files should be copied, unless they're
    2095             # bytecode files.
    2096             "somemodule.py": "import that"}
    2097         self.assertStructure(self.outputDir, outputStructure)
    2098 
    2099 
    2100     def test_stageFileHandlesEXDEV(self):
    2101         """
    2102         _stageFile should fall back to copying if os.link raises EXDEV.
    2103         """
    2104         def mock_link(src, dst):
    2105             raise OSError(errno.EXDEV, "dummy error")
    2106 
    2107         # Mock out os.link so that it always fails with EXDEV.
    2108         self.patch(os, "link", mock_link)
    2109 
    2110         # Staging a file should still work properly.
    2111 
    2112         # Make a test file
    2113         inputFile = self.rootDir.child("foo")
    2114         inputFile.setContent("bar")
    2115         modeReadOnly = stat.S_IRUSR|stat.S_IRGRP|stat.S_IROTH
    2116         inputFile.chmod(modeReadOnly)
    2117 
    2118         # Stage the file into an existing directory.
    2119         outputFile = self.outputDir.child("foo")
    2120         _stageFile(inputFile, outputFile)
    2121 
    2122         # Check the contents of the staged file
    2123         self.assertEquals(outputFile.getContent(), "bar")
    2124 
    2125         # Check the mode of the staged file
    2126         outputFile.restat()
    2127         self.assertEquals(
    2128             outputFile.statinfo.st_mode, (modeReadOnly | stat.S_IFREG))
    2129 
    2130     if not getattr(os, "link", None):
    2131         test_stageFileHandlesEXDEV.skip = "OS does not support hard-links"
    2132 
    2133 
    2134     def test_twistedDistributionExcludesWeb2AndVFSAndAdmin(self):
    2135         """
    2136         The main Twisted distribution does not include web2 or vfs, or the
    2137         bin/admin directory.
    2138         """
    2139         loreInput, loreOutput = self.getArbitraryLoreInputAndOutput("10.0.0")
    2140         coreIndexInput, coreIndexOutput = self.getArbitraryLoreInputAndOutput(
    2141             "10.0.0", prefix="howto/")
    2142 
    2143         structure = {
    2144             "README": "Twisted",
    2145             "unrelated": "x",
    2146             "LICENSE": "copyright!",
    2147             "setup.py": "import toplevel",
    2148             "bin": {"web2": {"websetroot": "SET ROOT"},
    2149                     "vfs": {"vfsitup": "hee hee"},
    2150                     "twistd": "TWISTD",
    2151                     "admin": {"build-a-thing": "yay"}},
    2152             "twisted":
    2153                 {"web2":
    2154                      {"__init__.py": "import WEB",
    2155                       "topfiles": {"setup.py": "import WEBINSTALL",
    2156                                    "README": "WEB!"}},
    2157                  "vfs":
    2158                      {"__init__.py": "import VFS",
    2159                       "blah blah": "blah blah"},
    2160                  "words": {"__init__.py": "import WORDS"},
    2161                  "plugins": {"twisted_web.py": "import WEBPLUG",
    2162                              "twisted_words.py": "import WORDPLUG",
    2163                              "twisted_web2.py": "import WEB2",
    2164                              "twisted_vfs.py": "import VFS"}},
    2165             "doc": {"web2": {"excluded!": "yay"},
    2166                     "vfs": {"unrelated": "whatever"},
    2167                     "core": {"howto": {"template.tpl": self.template},
    2168                              "index.xhtml": coreIndexInput}}}
    2169 
    2170         outStructure = {
    2171             "README": "Twisted",
    2172             "unrelated": "x",
    2173             "LICENSE": "copyright!",
    2174             "setup.py": "import toplevel",
    2175             "bin": {"twistd": "TWISTD"},
    2176             "twisted":
    2177                 {"words": {"__init__.py": "import WORDS"},
    2178                  "plugins": {"twisted_web.py": "import WEBPLUG",
    2179                              "twisted_words.py": "import WORDPLUG"}},
    2180             "doc": {"core": {"howto": {"template.tpl": self.template},
    2181                              "index.html": coreIndexOutput}}}
    2182         self.createStructure(self.rootDir, structure)
    2183 
    2184         outputFile = self.builder.buildTwisted("10.0.0")
    2185 
    2186         self.assertExtractedStructure(outputFile, outStructure)
    2187 
    2188 
    2189     def test_subProjectLayout(self):
    2190         """
    2191         The subproject tarball includes files like so:
    2192 
    2193         1. twisted/<subproject>/topfiles defines the files that will be in the
    2194            top level in the tarball, except LICENSE, which comes from the real
    2195            top-level directory.
    2196         2. twisted/<subproject> is included, but without the topfiles entry
    2197            in that directory. No other twisted subpackages are included.
    2198         3. twisted/plugins/twisted_<subproject>.py is included, but nothing
    2199            else in plugins is.
    2200         """
    2201         structure = {
    2202             "README": "HI!@",
    2203             "unrelated": "x",
    2204             "LICENSE": "copyright!",
    2205             "setup.py": "import toplevel",
    2206             "bin": {"web": {"websetroot": "SET ROOT"},
    2207                     "words": {"im": "#!im"}},
    2208             "twisted":
    2209                 {"web":
    2210                      {"__init__.py": "import WEB",
    2211                       "topfiles": {"setup.py": "import WEBINSTALL",
    2212                                    "README": "WEB!"}},
    2213                  "words": {"__init__.py": "import WORDS"},
    2214                  "plugins": {"twisted_web.py": "import WEBPLUG",
    2215                              "twisted_words.py": "import WORDPLUG"}}}
    2216 
    2217         outStructure = {
    2218             "README": "WEB!",
    2219             "LICENSE": "copyright!",
    2220             "setup.py": "import WEBINSTALL",
    2221             "bin": {"websetroot": "SET ROOT"},
    2222             "twisted": {"web": {"__init__.py": "import WEB"},
    2223                         "plugins": {"twisted_web.py": "import WEBPLUG"}}}
    2224 
    2225         self.createStructure(self.rootDir, structure)
    2226 
    2227         outputFile = self.builder.buildSubProject("web", "0.3.0")
    2228 
    2229         self.assertExtractedStructure(outputFile, outStructure)
    2230 
    2231 
    2232     def test_minimalSubProjectLayout(self):
    2233         """
    2234         buildSubProject should work with minimal subprojects.
    2235         """
    2236         structure = {
    2237             "LICENSE": "copyright!",
    2238             "bin": {},
    2239             "twisted":
    2240                 {"web": {"__init__.py": "import WEB",
    2241                          "topfiles": {"setup.py": "import WEBINSTALL"}},
    2242                  "plugins": {}}}
    2243 
    2244         outStructure = {
    2245             "setup.py": "import WEBINSTALL",
    2246             "LICENSE": "copyright!",
    2247             "twisted": {"web": {"__init__.py": "import WEB"}}}
    2248 
    2249         self.createStructure(self.rootDir, structure)
    2250 
    2251         outputFile = self.builder.buildSubProject("web", "0.3.0")
    2252 
    2253         self.assertExtractedStructure(outputFile, outStructure)
    2254 
    2255 
    2256     def test_subProjectDocBuilding(self):
    2257         """
    2258         When building a subproject release, documentation should be built with
    2259         lore.
    2260         """
    2261         loreInput, loreOutput = self.getArbitraryLoreInputAndOutput("0.3.0")
    2262         manInput = self.getArbitraryManInput()
    2263         manOutput = self.getArbitraryManHTMLOutput("0.3.0", "../howto/")
    2264         structure = {
    2265             "LICENSE": "copyright!",
    2266             "twisted": {"web": {"__init__.py": "import WEB",
    2267                                 "topfiles": {"setup.py": "import WEBINST"}}},
    2268             "doc": {"web": {"howto": {"index.xhtml": loreInput},
    2269                             "man": {"twistd.1": manInput}},
    2270                     "core": {"howto": {"template.tpl": self.template}}
    2271                     }
    2272             }
    2273 
    2274         outStructure = {
    2275             "LICENSE": "copyright!",
    2276             "setup.py": "import WEBINST",
    2277             "twisted": {"web": {"__init__.py": "import WEB"}},
    2278             "doc": {"howto": {"index.html": loreOutput},
    2279                     "man": {"twistd.1": manInput,
    2280                             "twistd-man.html": manOutput}}}
    2281 
    2282         self.createStructure(self.rootDir, structure)
    2283 
    2284         outputFile = self.builder.buildSubProject("web", "0.3.0")
    2285 
    2286         self.assertExtractedStructure(outputFile, outStructure)
    2287 
    2288 
    2289     def test_coreProjectLayout(self):
    2290         """
    2291         The core tarball looks a lot like a subproject tarball, except it
    2292         doesn't include:
    2293 
    2294         - Python packages from other subprojects
    2295         - plugins from other subprojects
    2296         - scripts from other subprojects
    2297         """
    2298         indexInput, indexOutput = self.getArbitraryLoreInputAndOutput(
    2299             "8.0.0", prefix="howto/")
    2300         howtoInput, howtoOutput = self.getArbitraryLoreInputAndOutput("8.0.0")
    2301         specInput, specOutput = self.getArbitraryLoreInputAndOutput(
    2302             "8.0.0", prefix="../howto/")
    2303         upgradeInput, upgradeOutput = self.getArbitraryLoreInputAndOutput(
    2304             "8.0.0", prefix="../howto/")
    2305         tutorialInput, tutorialOutput = self.getArbitraryLoreInputAndOutput(
    2306             "8.0.0", prefix="../")
    2307 
    2308         structure = {
    2309             "LICENSE": "copyright!",
    2310             "twisted": {"__init__.py": "twisted",
    2311                         "python": {"__init__.py": "python",
    2312                                    "roots.py": "roots!"},
    2313                         "conch": {"__init__.py": "conch",
    2314                                   "unrelated.py": "import conch"},
    2315                         "plugin.py": "plugin",
    2316                         "plugins": {"twisted_web.py": "webplug",
    2317                                     "twisted_whatever.py": "include!",
    2318                                     "cred.py": "include!"},
    2319                         "topfiles": {"setup.py": "import CORE",
    2320                                      "README": "core readme"}},
    2321             "doc": {"core": {"howto": {"template.tpl": self.template,
    2322                                        "index.xhtml": howtoInput,
    2323                                        "tutorial":
    2324                                            {"index.xhtml": tutorialInput}},
    2325                              "specifications": {"index.xhtml": specInput},
    2326                              "upgrades": {"index.xhtml": upgradeInput},
    2327                              "examples": {"foo.py": "foo.py"},
    2328                              "index.xhtml": indexInput},
    2329                     "web": {"howto": {"index.xhtml": "webindex"}}},
    2330             "bin": {"twistd": "TWISTD",
    2331                     "web": {"websetroot": "websetroot"}}
    2332             }
    2333 
    2334         outStructure = {
    2335             "LICENSE": "copyright!",
    2336             "setup.py": "import CORE",
    2337             "README": "core readme",
    2338             "twisted": {"__init__.py": "twisted",
    2339                         "python": {"__init__.py": "python",
    2340                                    "roots.py": "roots!"},
    2341                         "plugin.py": "plugin",
    2342                         "plugins": {"twisted_whatever.py": "include!",
    2343                                     "cred.py": "include!"}},
    2344             "doc": {"howto": {"template.tpl": self.template,
    2345                               "index.html": howtoOutput,
    2346                               "tutorial": {"index.html": tutorialOutput}},
    2347                     "specifications": {"index.html": specOutput},
    2348                     "upgrades": {"index.html": upgradeOutput},
    2349                     "examples": {"foo.py": "foo.py"},
    2350                     "index.html": indexOutput},
    2351             "bin": {"twistd": "TWISTD"},
    2352             }
    2353 
    2354         self.createStructure(self.rootDir, structure)
    2355         outputFile = self.builder.buildCore("8.0.0")
    2356         self.assertExtractedStructure(outputFile, outStructure)
    2357 
    2358 
    2359     def test_apiBaseURL(self):
    2360         """
    2361         DistributionBuilder builds documentation with the specified
    2362         API base URL.
    2363         """
    2364         apiBaseURL = "http://%s"
    2365         builder = DistributionBuilder(self.rootDir, self.outputDir,
    2366                                       apiBaseURL=apiBaseURL)
    2367         loreInput, loreOutput = self.getArbitraryLoreInputAndOutput(
    2368             "0.3.0", apiBaseURL=apiBaseURL)
    2369         structure = {
    2370             "LICENSE": "copyright!",
    2371             "twisted": {"web": {"__init__.py": "import WEB",
    2372                                 "topfiles": {"setup.py": "import WEBINST"}}},
    2373             "doc": {"web": {"howto": {"index.xhtml": loreInput}},
    2374                     "core": {"howto": {"template.tpl": self.template}}
    2375                     }
    2376             }
    2377 
    2378         outStructure = {
    2379             "LICENSE": "copyright!",
    2380             "setup.py": "import WEBINST",
    2381             "twisted": {"web": {"__init__.py": "import WEB"}},
    2382             "doc": {"howto": {"index.html": loreOutput}}}
    2383 
    2384         self.createStructure(self.rootDir, structure)
    2385         outputFile = builder.buildSubProject("web", "0.3.0")
    2386         self.assertExtractedStructure(outputFile, outStructure)
    2387 
    2388 
    2389 
    23901222class BuildAllTarballsTest(DistributionBuilderTestBase):
    23911223    """
    23921224    Tests for L{DistributionBuilder.buildAllTarballs}.
    class BuildAllTarballsTest(DistributionBuilderTestBase): 
    24031235        DistributionBuilderTestBase.tearDown(self)
    24041236
    24051237
    2406     def test_makeAPIBaseURLIsSubstitutable(self):
    2407         """
    2408         makeAPIBaseURL has a place to subtitute an API name.
    2409         """
    2410         url = makeAPIBaseURL("1.0")
    2411 
    2412         # Check that Python's string substitution actually works on this
    2413         # string.
    2414         url % ("foo",)
    2415 
    2416 
    24171238    def test_buildAllTarballs(self):
    24181239        """
    24191240        L{buildAllTarballs} builds tarballs for Twisted and all of its
    class ScriptTests(BuilderTestsMixin, StructureAssertingMixin, TestCase): 
    27161537
    27171538
    27181539
    2719 class IsDistributableTest(TestCase):
    2720 
    2721 
    2722     def test_fixedNamesExcluded(self):
    2723         """
    2724         isDistributable denies certain fixed names from being packaged.
    2725         """
    2726         self.assertEquals(isDistributable(FilePath("foo/CVS")), False)
    2727 
    2728 
    2729     def test_hiddenFilesExcluded(self):
    2730         """
    2731         isDistributable denies names beginning with a ".".
    2732         """
    2733         self.assertEquals(isDistributable(FilePath("foo/.svn")), False)
    2734 
    2735 
    2736     def test_byteCodeFilesExcluded(self):
    2737         """
    2738         isDistributable denies names Python bytecode files.
    2739         """
    2740         self.assertEquals(isDistributable(FilePath("foo/bar.pyc")), False)
    2741         self.assertEquals(isDistributable(FilePath("foo/bar.pyo")), False)
    2742 
    2743     def test_otherFilesIncluded(self):
    2744         """
    2745         isDistributable allows files with other names.
    2746         """
    2747         self.assertEquals(isDistributable(FilePath("foo/bar.py")), True)
    2748         self.assertEquals(isDistributable(FilePath("foo/README")), True)
    2749         self.assertEquals(isDistributable(FilePath("foo/twisted")), True)
  • twisted/topfiles/setup.py

    diff --git a/twisted/topfiles/setup.py b/twisted/topfiles/setup.py
    index 00dc801..16c16a5 100644
    a b if os.path.exists('twisted'): 
    1919from twisted import copyright
    2020from twisted.python.dist import setup, ConditionalExtension as Extension
    2121from twisted.python.dist import getPackages, getDataFiles, getScripts
    22 from twisted.python._release import twisted_subprojects
     22from twisted.python._dist import twisted_subprojects
    2323
    2424
    2525