Ticket #4138: sdist-support-4138.3.patch

setup.py

@@ -31 +31 @@
                 execfile(setup_py, ns, ns)
                 if "extensions" in ns:
                     extensions.extend(ns["extensions"])
-                    
+
     return extensions
 
 
@@ -43 +43 @@
     if os.path.exists('twisted'):
         sys.path.insert(0, '.')
     from twisted import copyright
-    from twisted.python.dist import getDataFiles, getScripts, getPackages, setup
+    from twisted.python.dist import getDataFiles, getScripts, getPackages
+    from twisted.python.dist import setup, _SDistTwisted
 
     # "" is included because core scripts are directly in bin/
     projects = [''] + [x for x in os.listdir('bin')
@@ -72 +73 @@
             packages = getPackages('twisted'),
             conditionalExtensions = getExtensions(),
             scripts = scripts,
-            data_files=getDataFiles('twisted'), 
+            data_files=getDataFiles('twisted'),
+            cmdclass = {'sdist': _SDistTwisted},
             )
 
     if 'setuptools' in sys.modules:

new file twisted/python/_dist.py

@@ +1 @@
+# -*- test-case-name: twisted.python.test.test__dist -*-
+# Copyright (c) 2010 Twisted Matrix Laboratories.
+# See LICENSE for details.
+"""
+Code to support distutils making a distribution of a release.
+
+Much of this code used to live in L{twisted.python._release}, but there is
+a distinction between a "distribution" and a "release". Only Twisted devs can
+make a release (using the code in C{t.p._release} to build API documentation,
+change version numbers, package tarballs and so forth), but anybody anywhere
+can use distutils to make a distribution of the files of a particular release.
+
+Because Twisted's release code is only designed to work in a POSIX environment,
+it's not appropriate for the generic distutils code to depend on it; therefore
+this module contains code for bundling the files of a release into
+a distribution, and both C{setup.py} and C{t.p._release} depend on it.
+"""
+
+import os, fnmatch, tarfile, errno, shutil
+from twisted.lore.scripts import lore
+
+
+twisted_subprojects = ["conch", "lore", "mail", "names",
+                       "news", "pair", "runner", "web", "web2",
+                       "words", "vfs"]
+
+
+
+# Files and directories matching these patterns will be excluded from Twisted
+# releases.
+EXCLUDE_PATTERNS = ["{arch}", "_darcs", "*.py[cdo]", "*.s[ol]", ".*", "*~"]
+
+
+
+def isDistributable(filepath):
+    """
+    Determine if the given item should be included in Twisted distributions.
+
+    This function is useful for filtering out files and directories in the
+    Twisted directory that aren't supposed to be part of the official Twisted
+    package - things like version control system metadata, editor backup files,
+    and various other detritus.
+
+    @type filepath: L{FilePath}
+    @param filepath: The file or directory that is a candidate for packaging.
+
+    @rtype: C{bool}
+    @return: True if the file should be included, False otherwise.
+    """
+    for pattern in EXCLUDE_PATTERNS:
+        if fnmatch.fnmatch(filepath.basename(), pattern):
+            return False
+    return True
+
+
+
+class NoDocumentsFound(Exception):
+    """
+    Raised when no input documents are found.
+    """
+
+
+
+class LoreBuilderMixin(object):
+    """
+    Base class for builders which invoke lore.
+    """
+    def lore(self, arguments):
+        """
+        Run lore with the given arguments.
+
+        @param arguments: A C{list} of C{str} giving command line arguments to
+            lore which should be used.
+        """
+        options = lore.Options()
+        options.parseOptions(["--null"] + arguments)
+        lore.runGivenOptions(options)
+
+
+
+class DocBuilder(LoreBuilderMixin):
+    """
+    Generate HTML documentation for projects.
+    """
+
+    def build(self, version, resourceDir, docDir, template, apiBaseURL=None,
+              deleteInput=False):
+        """
+        Build the documentation in C{docDir} with Lore.
+
+        Input files ending in .xhtml will be considered. Output will written as
+        .html files.
+
+        @param version: the version of the documentation to pass to lore.
+        @type version: C{str}
+
+        @param resourceDir: The directory which contains the toplevel index and
+            stylesheet file for this section of documentation.
+        @type resourceDir: L{twisted.python.filepath.FilePath}
+
+        @param docDir: The directory of the documentation.
+        @type docDir: L{twisted.python.filepath.FilePath}
+
+        @param template: The template used to generate the documentation.
+        @type template: L{twisted.python.filepath.FilePath}
+
+        @type apiBaseURL: C{str} or C{NoneType}
+        @param apiBaseURL: A format string which will be interpolated with the
+            fully-qualified Python name for each API link.  For example, to
+            generate the Twisted 8.0.0 documentation, pass
+            C{"http://twistedmatrix.com/documents/8.0.0/api/%s.html"}.
+
+        @param deleteInput: If True, the input documents will be deleted after
+            their output is generated.
+        @type deleteInput: C{bool}
+
+        @raise NoDocumentsFound: When there are no .xhtml files in the given
+            C{docDir}.
+        """
+        linkrel = self.getLinkrel(resourceDir, docDir)
+        inputFiles = docDir.globChildren("*.xhtml")
+        filenames = [x.path for x in inputFiles]
+        if not filenames:
+            raise NoDocumentsFound("No input documents found in %s" % (docDir,))
+        if apiBaseURL is not None:
+            arguments = ["--config", "baseurl=" + apiBaseURL]
+        else:
+            arguments = []
+        arguments.extend(["--config", "template=%s" % (template.path,),
+                          "--config", "ext=.html",
+                          "--config", "version=%s" % (version,),
+                          "--linkrel", linkrel] + filenames)
+        self.lore(arguments)
+        if deleteInput:
+            for inputFile in inputFiles:
+                inputFile.remove()
+
+
+    def getLinkrel(self, resourceDir, docDir):
+        """
+        Calculate a value appropriate for Lore's --linkrel option.
+
+        Lore's --linkrel option defines how to 'find' documents that are
+        linked to from TEMPLATE files (NOT document bodies). That is, it's a
+        prefix for links ('a' and 'link') in the template.
+
+        @param resourceDir: The directory which contains the toplevel index and
+            stylesheet file for this section of documentation.
+        @type resourceDir: L{twisted.python.filepath.FilePath}
+
+        @param docDir: The directory containing documents that must link to
+            C{resourceDir}.
+        @type docDir: L{twisted.python.filepath.FilePath}
+        """
+        if resourceDir != docDir:
+            return '/'.join(filePathDelta(docDir, resourceDir)) + "/"
+        else:
+            return ""
+
+
+
+class ManBuilder(LoreBuilderMixin):
+    """
+    Generate man pages of the different existing scripts.
+    """
+
+    def build(self, manDir):
+        """
+        Generate Lore input files from the man pages in C{manDir}.
+
+        Input files ending in .1 will be considered. Output will written as
+        -man.xhtml files.
+
+        @param manDir: The directory of the man pages.
+        @type manDir: L{twisted.python.filepath.FilePath}
+
+        @raise NoDocumentsFound: When there are no .1 files in the given
+            C{manDir}.
+        """
+        inputFiles = manDir.globChildren("*.1")
+        filenames = [x.path for x in inputFiles]
+        if not filenames:
+            raise NoDocumentsFound("No manual pages found in %s" % (manDir,))
+        arguments = ["--input", "man",
+                     "--output", "lore",
+                     "--config", "ext=-man.xhtml"] + filenames
+        self.lore(arguments)
+
+
+
+def _stageFile(src, dest):
+    """
+    Stages src at the destination path.
+
+    "Staging", in this case, means "creating a temporary copy to be archived".
+    In particular, we want to preserve all the metadata of the original file,
+    but we don't care about whether edits to the file propagate back and forth
+    (since the staged version should never be edited). We hard-link the file if
+    we can, otherwise we copy it and preserve metadata.
+
+    @type src: L{twisted.python.filepath.FilePath}
+    @param src: The file or path to be staged.
+
+    @type dest: L{twisted.python.filepath.FilePath}
+    @param dest: The path at which the source file should be staged. Any
+        missing directories in this path will be created.
+
+    @raise OSError: If the source is a file, and the destination already
+        exists, C{OSError} will be raised with the C{errno} attribute set to
+        C{EEXIST}.
+    """
+
+    if not isDistributable(src):
+        # Not a file we care about, quietly skip it.
+        return
+
+    if src.isfile():
+        # Make sure the directory's parent exists.
+        destParent = dest.parent()
+        if not destParent.exists():
+            destParent.makedirs()
+
+        # If the file already exists, raise an exception.
+        # os.link raises OSError, shutil.copy (sometimes) raises IOError or
+        # overwrites the destination, so let's do the checking ourselves and
+        # raise our own error.
+        if dest.exists():
+            raise OSError(errno.EEXIST, "File exists: %s" % (dest.path,))
+
+        # If we can create a hard link, that's faster than trying to copy
+        # things around.
+        if hasattr(os, "link"):
+            copyfunc = os.link
+        else:
+            copyfunc = shutil.copy2
+
+        try:
+            copyfunc(src.path, dest.path)
+        except OSError, e:
+            if e.errno == errno.EXDEV:
+                shutil.copy2(src.path, dest.path)
+            else:
+                raise
+
+    elif src.isdir():
+        if not dest.exists():
+            dest.makedirs()
+
+        for child in src.children():
+            _stageFile(child, dest.child(child.basename()))
+
+    else:
+        raise NotImplementedError("Can only stage files or directories")
+
+
+
+class DistributionBuilder(object):
+    """
+    A builder of Twisted distributions.
+
+    This knows how to build tarballs for Twisted and all of its subprojects.
+
+    @type blacklist: C{list} of C{str}
+    @cvar blacklist: The list subproject names to exclude from the main Twisted
+        tarball and for which no individual project tarballs will be built.
+    """
+
+    blacklist = ["vfs", "web2"]
+
+    def __init__(self, rootDirectory, outputDirectory, apiBaseURL=None):
+        """
+        Create a distribution builder.
+
+        @param rootDirectory: root of a Twisted export which will populate
+            subsequent tarballs.
+        @type rootDirectory: L{FilePath}.
+
+        @param outputDirectory: The directory in which to create the tarballs.
+        @type outputDirectory: L{FilePath}
+
+        @type apiBaseURL: C{str} or C{NoneType}
+        @param apiBaseURL: A format string which will be interpolated with the
+            fully-qualified Python name for each API link.  For example, to
+            generate the Twisted 8.0.0 documentation, pass
+            C{"http://twistedmatrix.com/documents/8.0.0/api/%s.html"}.
+        """
+        self.rootDirectory = rootDirectory
+        self.outputDirectory = outputDirectory
+        self.apiBaseURL = apiBaseURL
+        self.manBuilder = ManBuilder()
+        self.docBuilder = DocBuilder()
+
+
+    def _buildDocInDir(self, path, version, howtoPath):
+        """
+        Generate documentation in the given path, building man pages first if
+        necessary and swallowing errors (so that directories without lore
+        documentation in them are ignored).
+
+        @param path: The path containing documentation to build.
+        @type path: L{FilePath}
+        @param version: The version of the project to include in all generated
+            pages.
+        @type version: C{str}
+        @param howtoPath: The "resource path" as L{DocBuilder} describes it.
+        @type howtoPath: L{FilePath}
+        """
+        templatePath = self.rootDirectory.child("doc").child("core"
+            ).child("howto").child("template.tpl")
+        if path.basename() == "man":
+            self.manBuilder.build(path)
+        if path.isdir():
+            try:
+                self.docBuilder.build(version, howtoPath, path,
+                    templatePath, self.apiBaseURL, True)
+            except NoDocumentsFound:
+                pass
+
+
+    def buildTwistedFiles(self, version, releaseName):
+        """
+        Build a directory containing the main Twisted distribution.
+        """
+        # Make all the directories we'll need for copying things to.
+        distDirectory = self.outputDirectory.child(releaseName)
+        distBin = distDirectory.child("bin")
+        distTwisted = distDirectory.child("twisted")
+        distPlugins = distTwisted.child("plugins")
+        distDoc = distDirectory.child("doc")
+
+        for dir in (distDirectory, distBin, distTwisted, distPlugins, distDoc):
+            dir.makedirs()
+
+        # Now, this part is nasty.  We need to exclude blacklisted subprojects
+        # from the main Twisted distribution. This means we need to exclude
+        # their bin directories, their documentation directories, their
+        # plugins, and their python packages. Given that there's no "add all
+        # but exclude these particular paths" functionality in tarfile, we have
+        # to walk through all these directories and add things that *aren't*
+        # part of the blacklisted projects.
+
+        for binthing in self.rootDirectory.child("bin").children():
+            # bin/admin should also not be included.
+            if binthing.basename() not in self.blacklist + ["admin"]:
+                _stageFile(binthing, distBin.child(binthing.basename()))
+
+        bad_plugins = ["twisted_%s.py" % (blacklisted,)
+                       for blacklisted in self.blacklist]
+
+        for submodule in self.rootDirectory.child("twisted").children():
+            if submodule.basename() == "plugins":
+                for plugin in submodule.children():
+                    if plugin.basename() not in bad_plugins:
+                        _stageFile(plugin,
+                                distPlugins.child(plugin.basename()))
+            elif submodule.basename() not in self.blacklist:
+                _stageFile(submodule, distTwisted.child(submodule.basename()))
+
+        for docDir in self.rootDirectory.child("doc").children():
+            if docDir.basename() not in self.blacklist:
+                _stageFile(docDir, distDoc.child(docDir.basename()))
+
+        for toplevel in self.rootDirectory.children():
+            if not toplevel.isdir():
+                _stageFile(toplevel, distDirectory.child(toplevel.basename()))
+
+        # Generate docs in the distribution directory.
+        docPath = distDirectory.child("doc")
+        if docPath.isdir():
+            for subProjectDir in docPath.children():
+                if (subProjectDir.isdir()
+                    and subProjectDir.basename() not in self.blacklist):
+                    for child in subProjectDir.walk():
+                        self._buildDocInDir(child, version,
+                            subProjectDir.child("howto"))
+
+
+    def buildTwisted(self, version):
+        """
+        Build the main Twisted distribution in C{Twisted-<version>.tar.bz2}.
+
+        Projects listed in in L{blacklist} will not have their plugins, code,
+        documentation, or bin directories included.
+
+        bin/admin is also excluded.
+
+        @type version: C{str}
+        @param version: The version of Twisted to build.
+
+        @return: The tarball file.
+        @rtype: L{FilePath}.
+        """
+        releaseName = "Twisted-%s" % (version,)
+
+        outputTree = self.outputDirectory.child(releaseName)
+        outputFile = self.outputDirectory.child(releaseName + ".tar.bz2")
+
+        tarball = tarfile.TarFile.open(outputFile.path, 'w:bz2')
+        self.buildTwistedFiles(version, releaseName)
+        tarball.add(outputTree.path, releaseName)
+        tarball.close()
+
+        outputTree.remove()
+
+        return outputFile
+
+
+    def buildCore(self, version):
+        """
+        Build a core distribution in C{TwistedCore-<version>.tar.bz2}.
+
+        This is very similar to L{buildSubProject}, but core tarballs and the
+        input are laid out slightly differently.
+
+         - scripts are in the top level of the C{bin} directory.
+         - code is included directly from the C{twisted} directory, excluding
+           subprojects.
+         - all plugins except the subproject plugins are included.
+
+        @type version: C{str}
+        @param version: The version of Twisted to build.
+
+        @return: The tarball file.
+        @rtype: L{FilePath}.
+        """
+        releaseName = "TwistedCore-%s" % (version,)
+        outputFile = self.outputDirectory.child(releaseName + ".tar.bz2")
+        buildPath = lambda *args: '/'.join((releaseName,) + args)
+        tarball = self._createBasicSubprojectTarball(
+            "core", version, outputFile)
+
+        # Include the bin directory for the subproject.
+        for path in self.rootDirectory.child("bin").children():
+            if not path.isdir():
+                tarball.add(path.path, buildPath("bin", path.basename()))
+
+        # Include all files within twisted/ that aren't part of a subproject.
+        for path in self.rootDirectory.child("twisted").children():
+            if path.basename() == "plugins":
+                for plugin in path.children():
+                    for subproject in twisted_subprojects:
+                        if plugin.basename() == "twisted_%s.py" % (subproject,):
+                            break
+                    else:
+                        tarball.add(plugin.path,
+                                    buildPath("twisted", "plugins",
+                                              plugin.basename()))
+            elif not path.basename() in twisted_subprojects + ["topfiles"]:
+                tarball.add(path.path, buildPath("twisted", path.basename()))
+
+        tarball.add(self.rootDirectory.child("twisted").child("topfiles").path,
+                    releaseName)
+        tarball.close()
+
+        return outputFile
+
+
+    def buildSubProject(self, projectName, version):
+        """
+        Build a subproject distribution in
+        C{Twisted<Projectname>-<version>.tar.bz2}.
+
+        @type projectName: C{str}
+        @param projectName: The lowercase name of the subproject to build.
+        @type version: C{str}
+        @param version: The version of Twisted to build.
+
+        @return: The tarball file.
+        @rtype: L{FilePath}.
+        """
+        releaseName = "Twisted%s-%s" % (projectName.capitalize(), version)
+        outputFile = self.outputDirectory.child(releaseName + ".tar.bz2")
+        buildPath = lambda *args: '/'.join((releaseName,) + args)
+        subProjectDir = self.rootDirectory.child("twisted").child(projectName)
+
+        tarball = self._createBasicSubprojectTarball(projectName, version,
+                                                     outputFile)
+
+        tarball.add(subProjectDir.child("topfiles").path, releaseName)
+
+        # Include all files in the subproject package except for topfiles.
+        for child in subProjectDir.children():
+            name = child.basename()
+            if name != "topfiles":
+                tarball.add(
+                    child.path,
+                    buildPath("twisted", projectName, name))
+
+        pluginsDir = self.rootDirectory.child("twisted").child("plugins")
+        # Include the plugin for the subproject.
+        pluginFileName = "twisted_%s.py" % (projectName,)
+        pluginFile = pluginsDir.child(pluginFileName)
+        if pluginFile.exists():
+            tarball.add(pluginFile.path,
+                        buildPath("twisted", "plugins", pluginFileName))
+
+        # Include the bin directory for the subproject.
+        binPath = self.rootDirectory.child("bin").child(projectName)
+        if binPath.isdir():
+            tarball.add(binPath.path, buildPath("bin"))
+        tarball.close()
+
+        return outputFile
+
+
+    def _createBasicSubprojectTarball(self, projectName, version, outputFile):
+        """
+        Helper method to create and fill a tarball with things common between
+        subprojects and core.
+
+        @param projectName: The subproject's name.
+        @type projectName: C{str}
+        @param version: The version of the release.
+        @type version: C{str}
+        @param outputFile: The location of the tar file to create.
+        @type outputFile: L{FilePath}
+        """
+        releaseName = "Twisted%s-%s" % (projectName.capitalize(), version)
+        buildPath = lambda *args: '/'.join((releaseName,) + args)
+
+        tarball = tarfile.TarFile.open(outputFile.path, 'w:bz2')
+
+        tarball.add(self.rootDirectory.child("LICENSE").path,
+                    buildPath("LICENSE"))
+
+        docPath = self.rootDirectory.child("doc").child(projectName)
+
+        if docPath.isdir():
+            for child in docPath.walk():
+                self._buildDocInDir(child, version, docPath.child("howto"))
+            tarball.add(docPath.path, buildPath("doc"))
+
+        return tarball
+
+
+
+def makeAPIBaseURL(version):
+    """
+    Guess where the Twisted API docs for a given version will live.
+
+    @type version: C{str}
+    @param version: A URL-safe string containing a version number, such as
+        "10.0.0".
+    @rtype: C{str}
+    @return: A URL template pointing to the Twisted API docs for the given
+        version, ready to have the class, module or function name substituted
+        in.
+    """
+    return "http://twistedmatrix.com/documents/%s/api/%%s.html" % (version,)
+
+
+
+def filePathDelta(origin, destination):
+    """
+    Return a list of strings that represent C{destination} as a path relative
+    to C{origin}.
+
+    It is assumed that both paths represent directories, not files. That is to
+    say, the delta of L{twisted.python.filepath.FilePath} /foo/bar to
+    L{twisted.python.filepath.FilePath} /foo/baz will be C{../baz},
+    not C{baz}.
+
+    @type origin: L{twisted.python.filepath.FilePath}
+    @param origin: The origin of the relative path.
+
+    @type destination: L{twisted.python.filepath.FilePath}
+    @param destination: The destination of the relative path.
+    """
+    commonItems = 0
+    path1 = origin.path.split(os.sep)
+    path2 = destination.path.split(os.sep)
+    for elem1, elem2 in zip(path1, path2):
+        if elem1 == elem2:
+            commonItems += 1
+        else:
+            break
+    path = [".."] * (len(path1) - commonItems)
+    return path + path2[commonItems:]
+
+
+

twisted/python/_release.py

@@ -16 +16 @@
 from datetime import date
 import sys
 import os
-from tempfile import mkdtemp
 import tarfile
+import errno
+import shutil
+from tempfile import mkdtemp
 
 from subprocess import PIPE, STDOUT, Popen
 
 from twisted.python.versions import Version
 from twisted.python.filepath import FilePath
-from twisted.python.dist import twisted_subprojects
-
-# This import is an example of why you shouldn't use this module unless you're
-# radix
-try:
-    from twisted.lore.scripts import lore
-except ImportError:
-    pass
+from twisted.python._dist import LoreBuilderMixin, DistributionBuilder
+from twisted.python._dist import makeAPIBaseURL, twisted_subprojects
+from twisted.python._dist import ManBuilder, DocBuilder, NoDocumentsFound
+from twisted.python._dist import isDistributable
 
 # The offset between a year and the corresponding major version number.
 VERSION_OFFSET = 2000
@@ -109 +107 @@
     return Version(version.package, major, minor, 0)
 
 
+
 def changeAllProjectVersions(root, versionTemplate):
     """
     Change the version of all projects (including core and all subprojects).
@@ -139 +138 @@
 
 
 
-
 class Project(object):
     """
     A representation of a project that has a version.
@@ -213 +211 @@
         project.updateVersion(getNextVersion(project.getVersion(), now=now))
 
 
+
 def generateVersionFileData(version):
     """
     Generate the data to be placed into a _version.py file.
@@ -231 +230 @@
     return data
 
 
+
 def replaceProjectVersion(filename, newversion):
     """
     Write version specification code into the given filename, which
@@ -266 +266 @@
 
 
 
-class NoDocumentsFound(Exception):
-    """
-    Raised when no input documents are found.
-    """
-
-
-
-class LoreBuilderMixin(object):
-    """
-    Base class for builders which invoke lore.
-    """
-    def lore(self, arguments):
-        """
-        Run lore with the given arguments.
-
-        @param arguments: A C{list} of C{str} giving command line arguments to
-            lore which should be used.
-        """
-        options = lore.Options()
-        options.parseOptions(["--null"] + arguments)
-        lore.runGivenOptions(options)
-
-
-
-class DocBuilder(LoreBuilderMixin):
-    """
-    Generate HTML documentation for projects.
-    """
-
-    def build(self, version, resourceDir, docDir, template, apiBaseURL=None,
-              deleteInput=False):
-        """
-        Build the documentation in C{docDir} with Lore.
-
-        Input files ending in .xhtml will be considered. Output will written as
-        .html files.
-
-        @param version: the version of the documentation to pass to lore.
-        @type version: C{str}
-
-        @param resourceDir: The directory which contains the toplevel index and
-            stylesheet file for this section of documentation.
-        @type resourceDir: L{twisted.python.filepath.FilePath}
-
-        @param docDir: The directory of the documentation.
-        @type docDir: L{twisted.python.filepath.FilePath}
-
-        @param template: The template used to generate the documentation.
-        @type template: L{twisted.python.filepath.FilePath}
-
-        @type apiBaseURL: C{str} or C{NoneType}
-        @param apiBaseURL: A format string which will be interpolated with the
-            fully-qualified Python name for each API link.  For example, to
-            generate the Twisted 8.0.0 documentation, pass
-            C{"http://twistedmatrix.com/documents/8.0.0/api/%s.html"}.
-
-        @param deleteInput: If True, the input documents will be deleted after
-            their output is generated.
-        @type deleteInput: C{bool}
-
-        @raise NoDocumentsFound: When there are no .xhtml files in the given
-            C{docDir}.
-        """
-        linkrel = self.getLinkrel(resourceDir, docDir)
-        inputFiles = docDir.globChildren("*.xhtml")
-        filenames = [x.path for x in inputFiles]
-        if not filenames:
-            raise NoDocumentsFound("No input documents found in %s" % (docDir,))
-        if apiBaseURL is not None:
-            arguments = ["--config", "baseurl=" + apiBaseURL]
-        else:
-            arguments = []
-        arguments.extend(["--config", "template=%s" % (template.path,),
-                          "--config", "ext=.html",
-                          "--config", "version=%s" % (version,),
-                          "--linkrel", linkrel] + filenames)
-        self.lore(arguments)
-        if deleteInput:
-            for inputFile in inputFiles:
-                inputFile.remove()
-
-
-    def getLinkrel(self, resourceDir, docDir):
-        """
-        Calculate a value appropriate for Lore's --linkrel option.
-
-        Lore's --linkrel option defines how to 'find' documents that are
-        linked to from TEMPLATE files (NOT document bodies). That is, it's a
-        prefix for links ('a' and 'link') in the template.
-
-        @param resourceDir: The directory which contains the toplevel index and
-            stylesheet file for this section of documentation.
-        @type resourceDir: L{twisted.python.filepath.FilePath}
-
-        @param docDir: The directory containing documents that must link to
-            C{resourceDir}.
-        @type docDir: L{twisted.python.filepath.FilePath}
-        """
-        if resourceDir != docDir:
-            return '/'.join(filePathDelta(docDir, resourceDir)) + "/"
-        else:
-            return ""
-
-
-
-class ManBuilder(LoreBuilderMixin):
-    """
-    Generate man pages of the different existing scripts.
-    """
-
-    def build(self, manDir):
-        """
-        Generate Lore input files from the man pages in C{manDir}.
-
-        Input files ending in .1 will be considered. Output will written as
-        -man.xhtml files.
-
-        @param manDir: The directory of the man pages.
-        @type manDir: L{twisted.python.filepath.FilePath}
-
-        @raise NoDocumentsFound: When there are no .1 files in the given
-            C{manDir}.
-        """
-        inputFiles = manDir.globChildren("*.1")
-        filenames = [x.path for x in inputFiles]
-        if not filenames:
-            raise NoDocumentsFound("No manual pages found in %s" % (manDir,))
-        arguments = ["--input", "man",
-                     "--output", "lore",
-                     "--config", "ext=-man.xhtml"] + filenames
-        self.lore(arguments)
-
-
-
 class APIBuilder(object):
     """
     Generate API documentation from source files using
@@ -810 +676 @@
 
 
 
-def filePathDelta(origin, destination):
-    """
-    Return a list of strings that represent C{destination} as a path relative
-    to C{origin}.
-
-    It is assumed that both paths represent directories, not files. That is to
-    say, the delta of L{twisted.python.filepath.FilePath} /foo/bar to
-    L{twisted.python.filepath.FilePath} /foo/baz will be C{../baz},
-    not C{baz}.
-
-    @type origin: L{twisted.python.filepath.FilePath}
-    @param origin: The origin of the relative path.
-
-    @type destination: L{twisted.python.filepath.FilePath}
-    @param destination: The destination of the relative path.
-    """
-    commonItems = 0
-    path1 = origin.path.split(os.sep)
-    path2 = destination.path.split(os.sep)
-    for elem1, elem2 in zip(path1, path2):
-        if elem1 == elem2:
-            commonItems += 1
-        else:
-            break
-    path = [".."] * (len(path1) - commonItems)
-    return path + path2[commonItems:]
-
-
-
-class DistributionBuilder(object):
-    """
-    A builder of Twisted distributions.
-
-    This knows how to build tarballs for Twisted and all of its subprojects.
-
-    @type blacklist: C{list} of C{str}
-    @cvar blacklist: The list subproject names to exclude from the main Twisted
-        tarball and for which no individual project tarballs will be built.
-    """
-
-    from twisted.python.dist import twisted_subprojects as subprojects
-    blacklist = ["vfs", "web2"]
-
-    def __init__(self, rootDirectory, outputDirectory, apiBaseURL=None):
-        """
-        Create a distribution builder.
-
-        @param rootDirectory: root of a Twisted export which will populate
-            subsequent tarballs.
-        @type rootDirectory: L{FilePath}.
-
-        @param outputDirectory: The directory in which to create the tarballs.
-        @type outputDirectory: L{FilePath}
-
-        @type apiBaseURL: C{str} or C{NoneType}
-        @param apiBaseURL: A format string which will be interpolated with the
-            fully-qualified Python name for each API link.  For example, to
-            generate the Twisted 8.0.0 documentation, pass
-            C{"http://twistedmatrix.com/documents/8.0.0/api/%s.html"}.
-        """
-        self.rootDirectory = rootDirectory
-        self.outputDirectory = outputDirectory
-        self.apiBaseURL = apiBaseURL
-        self.manBuilder = ManBuilder()
-        self.docBuilder = DocBuilder()
-
-
-    def _buildDocInDir(self, path, version, howtoPath):
-        """
-        Generate documentation in the given path, building man pages first if
-        necessary and swallowing errors (so that directories without lore
-        documentation in them are ignored).
-
-        @param path: The path containing documentation to build.
-        @type path: L{FilePath}
-        @param version: The version of the project to include in all generated
-            pages.
-        @type version: C{str}
-        @param howtoPath: The "resource path" as L{DocBuilder} describes it.
-        @type howtoPath: L{FilePath}
-        """
-        templatePath = self.rootDirectory.child("doc").child("core"
-            ).child("howto").child("template.tpl")
-        if path.basename() == "man":
-            self.manBuilder.build(path)
-        if path.isdir():
-            try:
-                self.docBuilder.build(version, howtoPath, path,
-                    templatePath, self.apiBaseURL, True)
-            except NoDocumentsFound:
-                pass
-
-
-    def buildTwisted(self, version):
-        """
-        Build the main Twisted distribution in C{Twisted-<version>.tar.bz2}.
-
-        Projects listed in in L{blacklist} will not have their plugins, code,
-        documentation, or bin directories included.
-
-        bin/admin is also excluded.
-
-        @type version: C{str}
-        @param version: The version of Twisted to build.
-
-        @return: The tarball file.
-        @rtype: L{FilePath}.
-        """
-        releaseName = "Twisted-%s" % (version,)
-        buildPath = lambda *args: '/'.join((releaseName,) + args)
-
-        outputFile = self.outputDirectory.child(releaseName + ".tar.bz2")
-        tarball = tarfile.TarFile.open(outputFile.path, 'w:bz2')
-
-        docPath = self.rootDirectory.child("doc")
-
-        # Generate docs!
-        if docPath.isdir():
-            for subProjectDir in docPath.children():
-                if (subProjectDir.isdir()
-                    and subProjectDir.basename() not in self.blacklist):
-                    for child in subProjectDir.walk():
-                        self._buildDocInDir(child, version,
-                            subProjectDir.child("howto"))
-
-        # Now, this part is nasty.  We need to exclude blacklisted subprojects
-        # from the main Twisted distribution. This means we need to exclude
-        # their bin directories, their documentation directories, their
-        # plugins, and their python packages. Given that there's no "add all
-        # but exclude these particular paths" functionality in tarfile, we have
-        # to walk through all these directories and add things that *aren't*
-        # part of the blacklisted projects.
-
-        for binthing in self.rootDirectory.child("bin").children():
-            # bin/admin should also not be included.
-            if binthing.basename() not in self.blacklist + ["admin"]:
-                tarball.add(binthing.path,
-                            buildPath("bin", binthing.basename()))
-
-        bad_plugins = ["twisted_%s.py" % (blacklisted,)
-                       for blacklisted in self.blacklist]
-
-        for submodule in self.rootDirectory.child("twisted").children():
-            if submodule.basename() == "plugins":
-                for plugin in submodule.children():
-                    if plugin.basename() not in bad_plugins:
-                        tarball.add(plugin.path, buildPath("twisted", "plugins",
-                                                           plugin.basename()))
-            elif submodule.basename() not in self.blacklist:
-                tarball.add(submodule.path, buildPath("twisted",
-                                                      submodule.basename()))
-
-        for docDir in self.rootDirectory.child("doc").children():
-            if docDir.basename() not in self.blacklist:
-                tarball.add(docDir.path, buildPath("doc", docDir.basename()))
-
-        for toplevel in self.rootDirectory.children():
-            if not toplevel.isdir():
-                tarball.add(toplevel.path, buildPath(toplevel.basename()))
-
-        tarball.close()
-
-        return outputFile
-
-
-    def buildCore(self, version):
-        """
-        Build a core distribution in C{TwistedCore-<version>.tar.bz2}.
-
-        This is very similar to L{buildSubProject}, but core tarballs and the
-        input are laid out slightly differently.
-
-         - scripts are in the top level of the C{bin} directory.
-         - code is included directly from the C{twisted} directory, excluding
-           subprojects.
-         - all plugins except the subproject plugins are included.
-
-        @type version: C{str}
-        @param version: The version of Twisted to build.
-
-        @return: The tarball file.
-        @rtype: L{FilePath}.
-        """
-        releaseName = "TwistedCore-%s" % (version,)
-        outputFile = self.outputDirectory.child(releaseName + ".tar.bz2")
-        buildPath = lambda *args: '/'.join((releaseName,) + args)
-        tarball = self._createBasicSubprojectTarball(
-            "core", version, outputFile)
-
-        # Include the bin directory for the subproject.
-        for path in self.rootDirectory.child("bin").children():
-            if not path.isdir():
-                tarball.add(path.path, buildPath("bin", path.basename()))
-
-        # Include all files within twisted/ that aren't part of a subproject.
-        for path in self.rootDirectory.child("twisted").children():
-            if path.basename() == "plugins":
-                for plugin in path.children():
-                    for subproject in self.subprojects:
-                        if plugin.basename() == "twisted_%s.py" % (subproject,):
-                            break
-                    else:
-                        tarball.add(plugin.path,
-                                    buildPath("twisted", "plugins",
-                                              plugin.basename()))
-            elif not path.basename() in self.subprojects + ["topfiles"]:
-                tarball.add(path.path, buildPath("twisted", path.basename()))
-
-        tarball.add(self.rootDirectory.child("twisted").child("topfiles").path,
-                    releaseName)
-        tarball.close()
-
-        return outputFile
-
-
-    def buildSubProject(self, projectName, version):
-        """
-        Build a subproject distribution in
-        C{Twisted<Projectname>-<version>.tar.bz2}.
-
-        @type projectName: C{str}
-        @param projectName: The lowercase name of the subproject to build.
-        @type version: C{str}
-        @param version: The version of Twisted to build.
-
-        @return: The tarball file.
-        @rtype: L{FilePath}.
-        """
-        releaseName = "Twisted%s-%s" % (projectName.capitalize(), version)
-        outputFile = self.outputDirectory.child(releaseName + ".tar.bz2")
-        buildPath = lambda *args: '/'.join((releaseName,) + args)
-        subProjectDir = self.rootDirectory.child("twisted").child(projectName)
-
-        tarball = self._createBasicSubprojectTarball(projectName, version,
-                                                     outputFile)
-
-        tarball.add(subProjectDir.child("topfiles").path, releaseName)
-
-        # Include all files in the subproject package except for topfiles.
-        for child in subProjectDir.children():
-            name = child.basename()
-            if name != "topfiles":
-                tarball.add(
-                    child.path,
-                    buildPath("twisted", projectName, name))
-
-        pluginsDir = self.rootDirectory.child("twisted").child("plugins")
-        # Include the plugin for the subproject.
-        pluginFileName = "twisted_%s.py" % (projectName,)
-        pluginFile = pluginsDir.child(pluginFileName)
-        if pluginFile.exists():
-            tarball.add(pluginFile.path,
-                        buildPath("twisted", "plugins", pluginFileName))
-
-        # Include the bin directory for the subproject.
-        binPath = self.rootDirectory.child("bin").child(projectName)
-        if binPath.isdir():
-            tarball.add(binPath.path, buildPath("bin"))
-        tarball.close()
-
-        return outputFile
-
-
-    def _createBasicSubprojectTarball(self, projectName, version, outputFile):
-        """
-        Helper method to create and fill a tarball with things common between
-        subprojects and core.
-
-        @param projectName: The subproject's name.
-        @type projectName: C{str}
-        @param version: The version of the release.
-        @type version: C{str}
-        @param outputFile: The location of the tar file to create.
-        @type outputFile: L{FilePath}
-        """
-        releaseName = "Twisted%s-%s" % (projectName.capitalize(), version)
-        buildPath = lambda *args: '/'.join((releaseName,) + args)
-
-        tarball = tarfile.TarFile.open(outputFile.path, 'w:bz2')
-
-        tarball.add(self.rootDirectory.child("LICENSE").path,
-                    buildPath("LICENSE"))
-
-        docPath = self.rootDirectory.child("doc").child(projectName)
-
-        if docPath.isdir():
-            for child in docPath.walk():
-                self._buildDocInDir(child, version, docPath.child("howto"))
-            tarball.add(docPath.path, buildPath("doc"))
-
-        return tarball
-
-
-
 class UncleanWorkingDirectory(Exception):
     """
     Raised when the working directory of an SVN checkout is unclean.
     """
 
 
+
 class NotWorkingDirectory(Exception):
     """
     Raised when a directory does not appear to be an SVN working directory.
     """
 
 
+
 def buildAllTarballs(checkout, destination):
     """
     Build complete tarballs (including documentation) for Twisted and all
@@ -1150 +724 @@
     version = Project(twistedPath).getVersion()
     versionString = version.base()
 
-    apiBaseURL = "http://twistedmatrix.com/documents/%s/api/%%s.html" % (
-        versionString)
     if not destination.exists():
         destination.createDirectory()
-    db = DistributionBuilder(export, destination, apiBaseURL=apiBaseURL)
+    db = DistributionBuilder(export, destination,
+            apiBaseURL=makeAPIBaseURL(versionString))
 
     db.buildCore(versionString)
     for subproject in twisted_subprojects:
@@ -1166 +739 @@
     workPath.remove()
 
 
+
 class ChangeVersionsScript(object):
     """
     A thing for changing version numbers. See L{main}.

twisted/python/dist.py

@@ -6 +6 @@
 Maintainer: Christopher Armstrong
 """
 
-import sys, os
-from distutils.command import build_scripts, install_data, build_ext, build_py
+import os
+from distutils.command import (build_scripts, install_data, build_ext,
+                               sdist)
 from distutils.errors import CompileError
 from distutils import core
 from distutils.core import Extension
+from twisted.python.filepath import FilePath
+from twisted.python._dist import DistributionBuilder, makeAPIBaseURL
+from twisted.python._dist import isDistributable
 
-twisted_subprojects = ["conch", "lore", "mail", "names",
-                       "news", "pair", "runner", "web", "web2",
-                       "words", "vfs"]
 
 
 class ConditionalExtension(Extension):
@@ -46 +47 @@
     """
     return core.setup(**get_setup_args(**kw))
 
+
+
 def get_setup_args(**kw):
     if 'twisted_subproject' in kw:
         if 'twisted' not in os.listdir('.'):
@@ -73 +76 @@
             kw.setdefault('py_modules', []).extend(py_modules)
             del kw['plugins']
 
-    if 'cmdclass' not in kw:
-        kw['cmdclass'] = {
+    defaultCmdClasses = {
             'install_data': install_data_twisted,
             'build_scripts': build_scripts_twisted}
-        if sys.version_info[:3] < (2, 3, 0):
-            kw['cmdclass']['build_py'] = build_py_twisted
+
+    if 'cmdclass' in kw:
+        # Override our defaults with setup.py's custom cmdclasses
+        defaultCmdClasses.update(kw['cmdclass'])
+    kw['cmdclass'] = defaultCmdClasses
 
     if "conditionalExtensions" in kw:
         extensions = kw["conditionalExtensions"]
@@ -100 +105 @@
         kw.setdefault('cmdclass', {})['build_ext'] = my_build_ext
     return kw
 
+
+
 def getVersion(proj, base="twisted"):
     """
     Extract the version number for a given project.
@@ -120 +127 @@
     return ns['version'].base()
 
 
-# Names that are exluded from globbing results:
-EXCLUDE_NAMES = ["{arch}", "CVS", ".cvsignore", "_darcs",
-                 "RCS", "SCCS", ".svn"]
-EXCLUDE_PATTERNS = ["*.py[cdo]", "*.s[ol]", ".#*", "*~", "*.py"]
-
-import fnmatch
-
-def _filterNames(names):
-    """Given a list of file names, return those names that should be copied.
-    """
-    names = [n for n in names
-             if n not in EXCLUDE_NAMES]
-    # This is needed when building a distro from a working
-    # copy (likely a checkout) rather than a pristine export:
-    for pattern in EXCLUDE_PATTERNS:
-        names = [n for n in names
-                 if (not fnmatch.fnmatch(n, pattern))
-                 and (not n.endswith('.py'))]
-    return names
 
 def relativeTo(base, relativee):
     """
-    Gets 'relativee' relative to 'basepath'.
-
-    i.e.,
-
-    >>> relativeTo('/home/', '/home/radix/')
-    'radix'
-    >>> relativeTo('.', '/home/radix/Projects/Twisted') # curdir is /home/radix
-    'Projects/Twisted'
-
-    The 'relativee' must be a child of 'basepath'.
+    Converts the path to C{base} to a path to {relativee}.
+
+    >>> relativeTo('/home', '/home/radix')
+    '/home/radix'
+    >>> relativeTo('../radix/', '/home/radix/foo')
+    '../radix/foo'
+
+    @type base: C{str}
+    @param base: A filesystem path.
+    @type relativee: C{str}
+    @param relativee: A filesystem path that is a child of C{base}.
+    @rtype: C{str}
+    @return: A filesystem path to relativee.
     """
     basepath = os.path.abspath(base)
     relativee = os.path.abspath(relativee)
@@ -163 +154 @@
     raise ValueError("%s is not a subpath of %s" % (relativee, basepath))
 
 
+
 def getDataFiles(dname, ignore=None, parent=None):
     """
     Get all the data files that should be included in this distutils Project.
@@ -186 +178 @@
     result = []
     for directory, subdirectories, filenames in os.walk(dname):
         resultfiles = []
-        for exname in EXCLUDE_NAMES:
-            if exname in subdirectories:
-                subdirectories.remove(exname)
-        for ig in ignore:
-            if ig in subdirectories:
-                subdirectories.remove(ig)
-        for filename in _filterNames(filenames):
-            resultfiles.append(filename)
+        basePath = FilePath(os.path.join(dname, directory))
+
+        for subdir in subdirectories:
+            if subdir in ignore or not isDistributable(basePath.child(subdir)):
+                subdirectories.remove(subdir)
+
+        for filename in filenames:
+            if (isDistributable(basePath.child(filename))
+                    and not filename.endswith(".py")):
+                resultfiles.append(filename)
+
         if resultfiles:
+            # Sort our results so that tests pass regardless of the underlying
+            # filesystem's file order.
+            resultfiles.sort()
             result.append((relativeTo(parent, directory),
                            [relativeTo(parent,
                                        os.path.join(directory, filename))
                             for filename in resultfiles]))
     return result
 
+
+
 def getPackages(dname, pkgname=None, results=None, ignore=None, parent=None):
     """
     Get all packages which are under dname. This is necessary for
@@ -251 +251 @@
                   [os.path.join(scriptdir, x) for x in thingies])
 
 
-## Helpers and distutil tweaks
-
-class build_py_twisted(build_py.build_py):
-    """
-    Changes behavior in Python 2.2 to support simultaneous specification of
-    `packages' and `py_modules'.
-    """
-    def run(self):
-        if self.py_modules:
-            self.build_modules()
-        if self.packages:
-            self.build_packages()
-        self.byte_compile(self.get_outputs(include_bytecode=0))
-
 
+## Helpers and distutil tweaks
 
 class build_scripts_twisted(build_scripts.build_scripts):
     """Renames scripts so they end with '.py' on Windows."""
@@ -359 +346 @@
         self.compiler.announce("checking for %s ..." % header_name, 0)
         return self._compile_helper("#include <%s>\n" % header_name)
 
+
+class _SDistTwisted(sdist.sdist):
+    """
+    Build a Twisted source distribution like the official release scripts do.
+    """
+
+    def get_file_list(self):
+        """
+        Overridden to do nothing.
+
+        Twisted does not use a MANIFEST file.
+        """
+
+    def make_release_tree(self, basedir, _):
+        """
+        Overridden to call the official release scripts' functionality.
+
+        Builds a Twisted source distribution in the given directory.
+        """
+        if 'twisted' not in os.listdir('.'):
+            raise RuntimeError("Sorry, you need to run setup.py from the "
+                               "toplevel source directory.")
+
+        rootDirectory = FilePath(".")
+        outputDirectory = FilePath(".")
+        version = self.distribution.get_version()
+        builder = DistributionBuilder(rootDirectory, outputDirectory,
+                apiBaseURL=makeAPIBaseURL(version))
+        builder.buildTwistedFiles(version, basedir)
+
+        self.distribution.metadata.write_pkg_info(basedir)

new file twisted/python/test/test__dist.py

@@ +1 @@
+# Copyright (c) 2010 Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+import os, stat, errno, tarfile
+from xml.dom import minidom as dom
+from twisted.trial.unittest import TestCase
+from twisted.python.filepath import FilePath
+
+from twisted.python._dist import DocBuilder, ManBuilder, isDistributable
+from twisted.python._dist import makeAPIBaseURL, DistributionBuilder
+from twisted.python._dist import NoDocumentsFound, filePathDelta
+from twisted.python._dist import _stageFile
+
+
+
+# When we test that scripts are installed with the "correct" permissions, we
+# expect the "correct" permissions to be rwxr-xr-x
+SCRIPT_PERMS = (
+        stat.S_IRWXU # rwx
+        | stat.S_IRGRP | stat.S_IXGRP # r-x
+        | stat.S_IROTH | stat.S_IXOTH) # r-x
+
+
+
+# Check a bunch of dependencies to skip tests if necessary.
+try:
+    from twisted.lore.scripts import lore
+except ImportError:
+    loreSkip = "Lore is not present."
+else:
+    loreSkip = None
+
+
+
+class StructureAssertingMixin(object):
+    """
+    A mixin for L{TestCase} subclasses which provides some methods for asserting
+    the structure and contents of directories and files on the filesystem.
+    """
+    def createStructure(self, parent, dirDict, origRoot=None):
+        """
+        Create a set of directories and files given a dict defining their
+        structure.
+
+        @param parent: The directory in which to create the structure.  It must
+            already exist.
+        @type parent: L{FilePath}
+
+        @param dirDict: The dict defining the structure. Keys should be strings
+            naming files, values should be strings describing file contents OR
+            dicts describing subdirectories.  All files are written in binary
+            mode.  Any string values are assumed to describe text files and
+            will have their newlines replaced with the platform-native newline
+            convention.  For example::
+
+                {"foofile": "foocontents",
+                 "bardir": {"barfile": "bar\ncontents"}}
+        @type dirDict: C{dict}
+
+        @param origRoot: The directory provided as C{parent} in the original
+            invocation of C{createStructure}. Leave this as C{None}, it's used
+            in recursion.
+        @type origRoot: L{FilePath} or C{None}
+        """
+        if origRoot is None:
+            origRoot = parent
+
+        for x in dirDict:
+            child = parent.child(x)
+            if isinstance(dirDict[x], dict):
+                child.createDirectory()
+                self.createStructure(child, dirDict[x], origRoot=origRoot)
+
+                # If x is in a bin directory, make sure children
+                # representing files have the executable bit set.
+                if "bin" in child.segmentsFrom(origRoot):
+                    for script in [k for (k,v) in dirDict[x].items()
+                            if isinstance(v, basestring)]:
+                        scriptPath = child.child(script)
+                        scriptPath.chmod(SCRIPT_PERMS)
+
+            else:
+                child.setContent(dirDict[x].replace('\n', os.linesep))
+
+    def assertStructure(self, root, dirDict):
+        """
+        Assert that a directory is equivalent to one described by a dict.
+
+        @param root: The filesystem directory to compare.
+        @type root: L{FilePath}
+        @param dirDict: The dict that should describe the contents of the
+            directory. It should be the same structure as the C{dirDict}
+            parameter to L{createStructure}.
+        @type dirDict: C{dict}
+        """
+        children = [x.basename() for x in root.children()]
+        for x in dirDict:
+            child = root.child(x)
+            if isinstance(dirDict[x], dict):
+                self.assertTrue(child.isdir(), "%s is not a dir!"
+                                % (child.path,))
+                self.assertStructure(child, dirDict[x])
+
+                # If x is in a bin directory, make sure children
+                # representing files have the executable bit set.
+                if "/bin" in child.path:
+                    for script in [k for (k,v) in dirDict[x].items()
+                            if isinstance(v, basestring)]:
+                        scriptPath = child.child(script)
+                        scriptPath.restat()
+                        # What with SVN and umask and all that jazz, all we can
+                        # really check is that these scripts have at least one
+                        # executable bit set.
+                        self.assertTrue(scriptPath.statinfo.st_mode &
+                                (stat.S_IXUSR|stat.S_IXGRP|stat.S_IXOTH),
+                                "File %r should be executable"
+                                % (scriptPath.path,))
+            else:
+                a = child.getContent().replace(os.linesep, '\n')
+                self.assertEquals(a, dirDict[x], child.path)
+            children.remove(x)
+        if children:
+            self.fail("There were extra children in %s: %s"
+                      % (root.path, children))
+
+
+    def assertExtractedStructure(self, outputFile, dirDict):
+        """
+        Assert that a tarfile content is equivalent to one described by a dict.
+
+        @param outputFile: The tar file built by L{DistributionBuilder}.
+        @type outputFile: L{FilePath}.
+        @param dirDict: The dict that should describe the contents of the
+            directory. It should be the same structure as the C{dirDict}
+            parameter to L{createStructure}.
+        @type dirDict: C{dict}
+        """
+        tarFile = tarfile.TarFile.open(outputFile.path, "r:bz2")
+        extracted = FilePath(self.mktemp())
+        extracted.createDirectory()
+        for info in tarFile:
+            tarFile.extract(info, path=extracted.path)
+        self.assertStructure(extracted.children()[0], dirDict)
+
+
+
+class BuilderTestsMixin(object):
+    """
+    A mixin class which provides various methods for creating sample Lore input
+    and output.
+
+    @cvar template: The lore template that will be used to prepare sample
+    output.
+    @type template: C{str}
+
+    @ivar docCounter: A counter which is incremented every time input is
+        generated and which is included in the documents.
+    @type docCounter: C{int}
+    """
+    template = '''
+    <html>
+    <head><title>Yo:</title></head>
+    <body>
+    <div class="body" />
+    <a href="index.html">Index</a>
+    <span class="version">Version: </span>
+    </body>
+    </html>
+    '''
+
+    def setUp(self):
+        """
+        Initialize the doc counter which ensures documents are unique.
+        """
+        self.docCounter = 0
+
+
+    def assertXMLEqual(self, first, second):
+        """
+        Verify that two strings represent the same XML document.
+        """
+        self.assertEqual(
+            dom.parseString(first).toxml(),
+            dom.parseString(second).toxml())
+
+
+    def getArbitraryOutput(self, version, counter, prefix="", apiBaseURL="%s"):
+        """
+        Get the correct HTML output for the arbitrary input returned by
+        L{getArbitraryLoreInput} for the given parameters.
+
+        @param version: The version string to include in the output.
+        @type version: C{str}
+        @param counter: A counter to include in the output.
+        @type counter: C{int}
+        """
+        document = """\
+<?xml version="1.0"?><html>
+    <head><title>Yo:Hi! Title: %(count)d</title></head>
+    <body>
+    <div class="content">Hi! %(count)d<div class="API"><a href="%(foobarLink)s" title="foobar">foobar</a></div></div>
+    <a href="%(prefix)sindex.html">Index</a>
+    <span class="version">Version: %(version)s</span>
+    </body>
+    </html>"""
+        # Try to normalize irrelevant whitespace.
+        return dom.parseString(
+            document % {"count": counter, "prefix": prefix,
+                        "version": version,
+                        "foobarLink": apiBaseURL % ("foobar",)}).toxml('utf-8')
+
+
+    def getArbitraryLoreInput(self, counter):
+        """
+        Get an arbitrary, unique (for this test case) string of lore input.
+
+        @param counter: A counter to include in the input.
+        @type counter: C{int}
+        """
+        template = (
+            '<html>'
+            '<head><title>Hi! Title: %(count)s</title></head>'
+            '<body>'
+            'Hi! %(count)s'
+            '<div class="API">foobar</div>'
+            '</body>'
+            '</html>')
+        return template % {"count": counter}
+
+
+    def getArbitraryLoreInputAndOutput(self, version, prefix="",
+                                       apiBaseURL="%s"):
+        """
+        Get an input document along with expected output for lore run on that
+        output document, assuming an appropriately-specified C{self.template}.
+
+        @param version: A version string to include in the input and output.
+        @type version: C{str}
+        @param prefix: The prefix to include in the link to the index.
+        @type prefix: C{str}
+
+        @return: A two-tuple of input and expected output.
+        @rtype: C{(str, str)}.
+        """
+        self.docCounter += 1
+        return (self.getArbitraryLoreInput(self.docCounter),
+                self.getArbitraryOutput(version, self.docCounter,
+                                        prefix=prefix, apiBaseURL=apiBaseURL))
+
+
+    def getArbitraryManInput(self):
+        """
+        Get an arbitrary man page content.
+        """
+        return """.TH MANHOLE "1" "August 2001" "" ""
+.SH NAME
+manhole \- Connect to a Twisted Manhole service
+.SH SYNOPSIS
+.B manhole
+.SH DESCRIPTION
+manhole is a GTK interface to Twisted Manhole services. You can execute python
+code as if at an interactive Python console inside a running Twisted process
+with this."""
+
+
+    def getArbitraryManLoreOutput(self):
+        """
+        Get an arbitrary lore input document which represents man-to-lore
+        output based on the man page returned from L{getArbitraryManInput}
+        """
+        return """\
+<?xml version="1.0"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html><head>
+<title>MANHOLE.1</title></head>
+<body>
+
+<h1>MANHOLE.1</h1>
+
+<h2>NAME</h2>
+
+<p>manhole - Connect to a Twisted Manhole service
+</p>
+
+<h2>SYNOPSIS</h2>
+
+<p><strong>manhole</strong> </p>
+
+<h2>DESCRIPTION</h2>
+
+<p>manhole is a GTK interface to Twisted Manhole services. You can execute python
+code as if at an interactive Python console inside a running Twisted process
+with this.</p>
+
+</body>
+</html>
+"""
+
+    def getArbitraryManHTMLOutput(self, version, prefix=""):
+        """
+        Get an arbitrary lore output document which represents the lore HTML
+        output based on the input document returned from
+        L{getArbitraryManLoreOutput}.
+
+        @param version: A version string to include in the document.
+        @type version: C{str}
+        @param prefix: The prefix to include in the link to the index.
+        @type prefix: C{str}
+        """
+        # Try to normalize the XML a little bit.
+        return dom.parseString("""\
+<?xml version="1.0" ?><html>
+    <head><title>Yo:MANHOLE.1</title></head>
+    <body>
+    <div class="content">
+
+<span/>
+
+<h2>NAME<a name="auto0"/></h2>
+
+<p>manhole - Connect to a Twisted Manhole service
+</p>
+
+<h2>SYNOPSIS<a name="auto1"/></h2>
+
+<p><strong>manhole</strong> </p>
+
+<h2>DESCRIPTION<a name="auto2"/></h2>
+
+<p>manhole is a GTK interface to Twisted Manhole services. You can execute python
+code as if at an interactive Python console inside a running Twisted process
+with this.</p>
+
+</div>
+    <a href="%(prefix)sindex.html">Index</a>
+    <span class="version">Version: %(version)s</span>
+    </body>
+    </html>""" % {
+            'prefix': prefix, 'version': version}).toxml("utf-8")
+
+
+
+class DocBuilderTestCase(TestCase, BuilderTestsMixin):
+    """
+    Tests for L{DocBuilder}.
+
+    Note for future maintainers: The exact byte equality assertions throughout
+    this suite may need to be updated due to minor differences in lore. They
+    should not be taken to mean that Lore must maintain the same byte format
+    forever. Feel free to update the tests when Lore changes, but please be
+    careful.
+    """
+    skip = loreSkip
+
+    def setUp(self):
+        """
+        Set up a few instance variables that will be useful.
+
+        @ivar builder: A plain L{DocBuilder}.
+        @ivar docCounter: An integer to be used as a counter by the
+            C{getArbitrary...} methods.
+        @ivar howtoDir: A L{FilePath} representing a directory to be used for
+            containing Lore documents.
+        @ivar templateFile: A L{FilePath} representing a file with
+            C{self.template} as its content.
+        """
+        BuilderTestsMixin.setUp(self)
+        self.builder = DocBuilder()
+        self.howtoDir = FilePath(self.mktemp())
+        self.howtoDir.createDirectory()
+        self.templateFile = self.howtoDir.child("template.tpl")
+        self.templateFile.setContent(self.template)
+
+
+    def test_build(self):
+        """
+        The L{DocBuilder} runs lore on all .xhtml files within a directory.
+        """
+        version = "1.2.3"
+        input1, output1 = self.getArbitraryLoreInputAndOutput(version)
+        input2, output2 = self.getArbitraryLoreInputAndOutput(version)
+
+        self.howtoDir.child("one.xhtml").setContent(input1)
+        self.howtoDir.child("two.xhtml").setContent(input2)
+
+        self.builder.build(version, self.howtoDir, self.howtoDir,
+                           self.templateFile)
+        out1 = self.howtoDir.child('one.html')
+        out2 = self.howtoDir.child('two.html')
+        self.assertXMLEqual(out1.getContent(), output1)
+        self.assertXMLEqual(out2.getContent(), output2)
+
+
+    def test_noDocumentsFound(self):
+        """
+        The C{build} method raises L{NoDocumentsFound} if there are no
+        .xhtml files in the given directory.
+        """
+        self.assertRaises(
+            NoDocumentsFound,
+            self.builder.build, "1.2.3", self.howtoDir, self.howtoDir,
+            self.templateFile)
+
+
+    def test_parentDocumentLinking(self):
+        """
+        The L{DocBuilder} generates correct links from documents to
+        template-generated links like stylesheets and index backreferences.
+        """
+        input = self.getArbitraryLoreInput(0)
+        tutoDir = self.howtoDir.child("tutorial")
+        tutoDir.createDirectory()
+        tutoDir.child("child.xhtml").setContent(input)
+        self.builder.build("1.2.3", self.howtoDir, tutoDir, self.templateFile)
+        outFile = tutoDir.child('child.html')
+        self.assertIn('<a href="../index.html">Index</a>',
+                      outFile.getContent())
+
+
+    def test_siblingDirectoryDocumentLinking(self):
+        """
+        It is necessary to generate documentation in a directory foo/bar where
+        stylesheet and indexes are located in foo/baz. Such resources should be
+        appropriately linked to.
+        """
+        input = self.getArbitraryLoreInput(0)
+        resourceDir = self.howtoDir.child("resources")
+        docDir = self.howtoDir.child("docs")
+        docDir.createDirectory()
+        docDir.child("child.xhtml").setContent(input)
+        self.builder.build("1.2.3", resourceDir, docDir, self.templateFile)
+        outFile = docDir.child('child.html')
+        self.assertIn('<a href="../resources/index.html">Index</a>',
+                      outFile.getContent())
+
+
+    def test_apiLinking(self):
+        """
+        The L{DocBuilder} generates correct links from documents to API
+        documentation.
+        """
+        version = "1.2.3"
+        input, output = self.getArbitraryLoreInputAndOutput(version)
+        self.howtoDir.child("one.xhtml").setContent(input)
+
+        self.builder.build(version, self.howtoDir, self.howtoDir,
+                           self.templateFile, "scheme:apilinks/%s.ext")
+        out = self.howtoDir.child('one.html')
+        self.assertIn(
+            '<a href="scheme:apilinks/foobar.ext" title="foobar">foobar</a>',
+            out.getContent())
+
+
+    def test_deleteInput(self):
+        """
+        L{DocBuilder.build} can be instructed to delete the input files after
+        generating the output based on them.
+        """
+        input1 = self.getArbitraryLoreInput(0)
+        self.howtoDir.child("one.xhtml").setContent(input1)
+        self.builder.build("whatever", self.howtoDir, self.howtoDir,
+                           self.templateFile, deleteInput=True)
+        self.assertTrue(self.howtoDir.child('one.html').exists())
+        self.assertFalse(self.howtoDir.child('one.xhtml').exists())
+
+
+    def test_doNotDeleteInput(self):
+        """
+        Input will not be deleted by default.
+        """
+        input1 = self.getArbitraryLoreInput(0)
+        self.howtoDir.child("one.xhtml").setContent(input1)
+        self.builder.build("whatever", self.howtoDir, self.howtoDir,
+                           self.templateFile)
+        self.assertTrue(self.howtoDir.child('one.html').exists())
+        self.assertTrue(self.howtoDir.child('one.xhtml').exists())
+
+
+    def test_getLinkrelToSameDirectory(self):
+        """
+        If the doc and resource directories are the same, the linkrel should be
+        an empty string.
+        """
+        linkrel = self.builder.getLinkrel(FilePath("/foo/bar"),
+                                          FilePath("/foo/bar"))
+        self.assertEquals(linkrel, "")
+
+
+    def test_getLinkrelToParentDirectory(self):
+        """
+        If the doc directory is a child of the resource directory, the linkrel
+        should make use of '..'.
+        """
+        linkrel = self.builder.getLinkrel(FilePath("/foo"),
+                                          FilePath("/foo/bar"))
+        self.assertEquals(linkrel, "../")
+
+
+    def test_getLinkrelToSibling(self):
+        """
+        If the doc directory is a sibling of the resource directory, the
+        linkrel should make use of '..' and a named segment.
+        """
+        linkrel = self.builder.getLinkrel(FilePath("/foo/howto"),
+                                          FilePath("/foo/examples"))
+        self.assertEquals(linkrel, "../howto/")
+
+
+    def test_getLinkrelToUncle(self):
+        """
+        If the doc directory is a sibling of the parent of the resource
+        directory, the linkrel should make use of multiple '..'s and a named
+        segment.
+        """
+        linkrel = self.builder.getLinkrel(FilePath("/foo/howto"),
+                                          FilePath("/foo/examples/quotes"))
+        self.assertEquals(linkrel, "../../howto/")
+
+
+
+class ManBuilderTestCase(TestCase, BuilderTestsMixin):
+    """
+    Tests for L{ManBuilder}.
+    """
+    skip = loreSkip
+
+    def setUp(self):
+        """
+        Set up a few instance variables that will be useful.
+
+        @ivar builder: A plain L{ManBuilder}.
+        @ivar manDir: A L{FilePath} representing a directory to be used for
+            containing man pages.
+        """
+        BuilderTestsMixin.setUp(self)
+        self.builder = ManBuilder()
+        self.manDir = FilePath(self.mktemp())
+        self.manDir.createDirectory()
+
+
+    def test_noDocumentsFound(self):
+        """
+        L{ManBuilder.build} raises L{NoDocumentsFound} if there are no
+        .1 files in the given directory.
+        """
+        self.assertRaises(NoDocumentsFound, self.builder.build, self.manDir)
+
+
+    def test_build(self):
+        """
+        Check that L{ManBuilder.build} find the man page in the directory, and
+        successfully produce a Lore content.
+        """
+        manContent = self.getArbitraryManInput()
+        self.manDir.child('test1.1').setContent(manContent)
+        self.builder.build(self.manDir)
+        output = self.manDir.child('test1-man.xhtml').getContent()
+        expected = self.getArbitraryManLoreOutput()
+        # No-op on *nix, fix for windows
+        expected = expected.replace('\n', os.linesep)
+        self.assertEquals(output, expected)
+
+
+    def test_toHTML(self):
+        """
+        Check that the content output by C{build} is compatible as input of
+        L{DocBuilder.build}.
+        """
+        manContent = self.getArbitraryManInput()
+        self.manDir.child('test1.1').setContent(manContent)
+        self.builder.build(self.manDir)
+
+        templateFile = self.manDir.child("template.tpl")
+        templateFile.setContent(DocBuilderTestCase.template)
+        docBuilder = DocBuilder()
+        docBuilder.build("1.2.3", self.manDir, self.manDir,
+                         templateFile)
+        output = self.manDir.child('test1-man.html').getContent()
+
+        self.assertXMLEqual(
+            output,
+            """\
+<?xml version="1.0" ?><html>
+    <head><title>Yo:MANHOLE.1</title></head>
+    <body>
+    <div class="content">
+
+<span/>
+
+<h2>NAME<a name="auto0"/></h2>
+
+<p>manhole - Connect to a Twisted Manhole service
+</p>
+
+<h2>SYNOPSIS<a name="auto1"/></h2>
+
+<p><strong>manhole</strong> </p>
+
+<h2>DESCRIPTION<a name="auto2"/></h2>
+
+<p>manhole is a GTK interface to Twisted Manhole services. You can execute python
+code as if at an interactive Python console inside a running Twisted process
+with this.</p>
+
+</div>
+    <a href="index.html">Index</a>
+    <span class="version">Version: 1.2.3</span>
+    </body>
+    </html>""")
+
+
+
+class DistributionBuilderTestBase(BuilderTestsMixin, StructureAssertingMixin,
+                                   TestCase):
+    """
+    Base for tests of L{DistributionBuilder}.
+    """
+    skip = loreSkip
+
+    def setUp(self):
+        BuilderTestsMixin.setUp(self)
+
+        self.rootDir = FilePath(self.mktemp())
+        self.rootDir.createDirectory()
+
+        self.outputDir = FilePath(self.mktemp())
+        self.outputDir.createDirectory()
+        self.builder = DistributionBuilder(self.rootDir, self.outputDir)
+
+
+
+class DistributionBuilderTest(DistributionBuilderTestBase):
+
+    def test_twistedDistribution(self):
+        """
+        The Twisted tarball contains everything in the source checkout, with
+        built documentation.
+        """
+        loreInput, loreOutput = self.getArbitraryLoreInputAndOutput("10.0.0")
+        manInput1 = self.getArbitraryManInput()
+        manOutput1 = self.getArbitraryManHTMLOutput("10.0.0", "../howto/")
+        manInput2 = self.getArbitraryManInput()
+        manOutput2 = self.getArbitraryManHTMLOutput("10.0.0", "../howto/")
+        coreIndexInput, coreIndexOutput = self.getArbitraryLoreInputAndOutput(
+            "10.0.0", prefix="howto/")
+
+        structure = {
+            "README": "Twisted",
+            "unrelated": "x",
+            "LICENSE": "copyright!",
+            "setup.py": "import toplevel",
+            "bin": {"web": {"websetroot": "SET ROOT"},
+                    "twistd": "TWISTD"},
+            "twisted":
+                {"web":
+                     {"__init__.py": "import WEB",
+                      "topfiles": {"setup.py": "import WEBINSTALL",
+                                   "README": "WEB!"}},
+                 "words": {"__init__.py": "import WORDS"},
+                 "plugins": {"twisted_web.py": "import WEBPLUG",
+                             "twisted_words.py": "import WORDPLUG"}},
+            "doc": {"web": {"howto": {"index.xhtml": loreInput},
+                            "man": {"websetroot.1": manInput2}},
+                    "core": {"howto": {"template.tpl": self.template},
+                             "man": {"twistd.1": manInput1},
+                             "index.xhtml": coreIndexInput}}}
+
+        outStructure = {
+            "README": "Twisted",
+            "unrelated": "x",
+            "LICENSE": "copyright!",
+            "setup.py": "import toplevel",
+            "bin": {"web": {"websetroot": "SET ROOT"},
+                    "twistd": "TWISTD"},
+            "twisted":
+                {"web": {"__init__.py": "import WEB",
+                         "topfiles": {"setup.py": "import WEBINSTALL",
+                                      "README": "WEB!"}},
+                 "words": {"__init__.py": "import WORDS"},
+                 "plugins": {"twisted_web.py": "import WEBPLUG",
+                             "twisted_words.py": "import WORDPLUG"}},
+            "doc": {"web": {"howto": {"index.html": loreOutput},
+                            "man": {"websetroot.1": manInput2,
+                                    "websetroot-man.html": manOutput2}},
+                    "core": {"howto": {"template.tpl": self.template},
+                             "man": {"twistd.1": manInput1,
+                                     "twistd-man.html": manOutput1},
+                             "index.html": coreIndexOutput}}}
+
+        self.createStructure(self.rootDir, structure)
+
+        outputFile = self.builder.buildTwisted("10.0.0")
+
+        self.assertExtractedStructure(outputFile, outStructure)
+
+
+    def test_stageFileStagesFiles(self):
+        """
+        L{_stageFile} duplicates the content and metadata of the given file.
+        """
+        # Make a test file
+        inputFile = self.rootDir.child("foo")
+
+        # Put some content in it.
+        inputFile.setContent("bar")
+
+        # Put a funny mode on it.
+        modeReadOnly = stat.S_IRUSR|stat.S_IRGRP|stat.S_IROTH
+        inputFile.chmod(modeReadOnly)
+
+        # Test the first: stage the file into an existing directory.
+        outputFile1 = self.outputDir.child("foo")
+
+        # Test the second: stage the file into a new directory.
+        outputFile2 = self.outputDir.preauthChild("sub/dir/foo")
+
+        for outputFile in [outputFile1, outputFile2]:
+            _stageFile(inputFile, outputFile)
+
+            # Check the contents of the staged file
+            self.failUnlessEqual(outputFile.open("r").read(), "bar")
+
+            # Check the mode of the staged file
+            outputFile.restat()
+            self.assertEqual(outputFile.statinfo.st_mode,
+                    (modeReadOnly | stat.S_IFREG))
+
+
+    def test_stageFileWillNotOverwrite(self):
+        """
+        L{_stageFile} raises an exception if asked to overwrite an output file.
+        """
+        # Make a test file
+        inputFile = self.rootDir.child("foo")
+        inputFile.setContent("bar")
+
+        # Make an output file.
+        outputFile = self.outputDir.child("foo")
+
+        # First attempt should work fine.
+        _stageFile(inputFile, outputFile)
+
+        # Second attempt should raise OSError with EEXIST.
+        exception = self.failUnlessRaises(OSError, _stageFile, inputFile,
+                outputFile)
+
+        self.failUnlessEqual(exception.errno, errno.EEXIST)
+
+
+    def test_stageFileStagesDirectories(self):
+        """
+        L{_stageFile} duplicates the content of the given directory.
+        """
+        # Make a test directory with stuff in it.
+        structure = {
+            "somedirectory": {
+                "somefile": "some content",
+                "anotherdirectory": {
+                    "anotherfile": "other content"}}}
+        self.createStructure(self.rootDir, structure)
+        inputDirectory = self.rootDir.child("somedirectory")
+
+        # Stage this directory structure
+        outputDirectory = self.outputDir.child("somedirectory")
+        _stageFile(inputDirectory, outputDirectory)
+
+        # Check that everything was copied across properly.
+        self.assertStructure(self.outputDir, structure)
+
+
+    def test_stageFileFiltersBytecode(self):
+        """
+        L{_stageFile} ignores Python bytecode files.
+        """
+        # Make a test directory with stuff in it.
+        inputStructure = {
+            "somepackage": {
+                "__init__.py": "",
+                "__init__.pyc": "gibberish",
+                "__init__.pyo": "more gibberish",
+                "module.py": "import this",
+                "module.pyc": "extra gibberish",
+                "module.pyo": "bonus gibberish",
+                "datafile.xqz": "A file with an unknown extension"},
+            "somemodule.py": "import that",
+            "somemodule.pyc": "surplus gibberish",
+            "somemodule.pyo": "sundry gibberish"}
+        self.createStructure(self.rootDir, inputStructure)
+
+        # Stage this directory structure
+        for child in self.rootDir.children():
+            dest = self.outputDir.child(child.basename())
+            _stageFile(child, dest)
+
+        # Check that everything but bytecode files has been copied across.
+        outputStructure = {
+            "somepackage": {
+                # Ordinary Python files should be copied.
+                "__init__.py": "",
+                "module.py": "import this",
+
+                # .pyc and .pyc files should be ignored.
+
+                # Other unknown files should be copied too.
+                "datafile.xqz": "A file with an unknown extension"},
+            # Individually staged files should be copied, unless they're
+            # bytecode files.
+            "somemodule.py": "import that"}
+        self.assertStructure(self.outputDir, outputStructure)
+
+
+    def test_stageFileFiltersVCSMetadata(self):
+        """
+        L{_stageFile} ignores common VCS directories.
+        """
+        # Make a test directory with stuff in it.
+        inputStructure = {
+            # Twisted's official repository is Subversion.
+            ".svn": {
+                "svn-data": "some Subversion data"},
+            # Twisted has a semi-official bzr mirror of the svn repository.
+            ".bzr": {
+                "bzr-data": "some Bazaar data"},
+            # git-svn is a popular way for git users to deal with svn
+            # repositories.
+            ".git": {
+                "git-data": "some Git data"},
+            "somepackage": {
+                # Subversion litters its .svn directories everywhere, not just
+                # the top-level.
+                ".svn": {
+                    "svn-data": "more Subversion data"},
+                "__init__.py": "",
+                "module.py": "import this"},
+            "somemodule.py": "import that"}
+        self.createStructure(self.rootDir, inputStructure)
+
+        # Stage this directory structure
+        for child in self.rootDir.children():
+            dest = self.outputDir.child(child.basename())
+            _stageFile(child, dest)
+
+        # Check that everything but VCS files has been copied across.
+        outputStructure = {
+            # No VCS files in the root.
+            "somepackage": {
+                # Ordinary Python files should be copied.
+                "__init__.py": "",
+                "module.py": "import this",
+
+                # No VCS files in the package, either.
+                },
+
+            # Individually staged files should be copied, unless they're
+            # bytecode files.
+            "somemodule.py": "import that"}
+        self.assertStructure(self.outputDir, outputStructure)
+
+
+    def test_stageFileHandlesEXDEV(self):
+        """
+        L{_stageFile} should fall back to copying if os.link raises EXDEV.
+        """
+        def mock_link(src, dst):
+            raise OSError(errno.EXDEV, "dummy error")
+
+        # Mock out os.link so that it always fails with EXDEV.
+        self.patch(os, "link", mock_link)
+
+        # Staging a file should still work properly.
+
+        # Make a test file
+        inputFile = self.rootDir.child("foo")
+        inputFile.setContent("bar")
+        modeReadOnly = stat.S_IRUSR|stat.S_IRGRP|stat.S_IROTH
+        inputFile.chmod(modeReadOnly)
+
+        # Stage the file into an existing directory.
+        outputFile = self.outputDir.child("foo")
+        _stageFile(inputFile, outputFile)
+
+        # Check the contents of the staged file
+        self.failUnlessEqual(outputFile.open("r").read(), "bar")
+
+        # Check the mode of the staged file
+        outputFile.restat()
+        self.assertEqual(outputFile.statinfo.st_mode,
+                (modeReadOnly | stat.S_IFREG))
+
+    if not getattr(os, "link", None):
+        test_stageFileHandlesEXDEV.skip = "OS does not support hard-links"
+
+
+    def test_twistedDistributionExcludesWeb2AndVFSAndAdmin(self):
+        """
+        The main Twisted distribution does not include web2 or vfs, or the
+        bin/admin directory.
+        """
+        loreInput, loreOutput = self.getArbitraryLoreInputAndOutput("10.0.0")
+        coreIndexInput, coreIndexOutput = self.getArbitraryLoreInputAndOutput(
+            "10.0.0", prefix="howto/")
+
+        structure = {
+            "README": "Twisted",
+            "unrelated": "x",
+            "LICENSE": "copyright!",
+            "setup.py": "import toplevel",
+            "bin": {"web2": {"websetroot": "SET ROOT"},
+                    "vfs": {"vfsitup": "hee hee"},
+                    "twistd": "TWISTD",
+                    "admin": {"build-a-thing": "yay"}},
+            "twisted":
+                {"web2":
+                     {"__init__.py": "import WEB",
+                      "topfiles": {"setup.py": "import WEBINSTALL",
+                                   "README": "WEB!"}},
+                 "vfs":
+                     {"__init__.py": "import VFS",
+                      "blah blah": "blah blah"},
+                 "words": {"__init__.py": "import WORDS"},
+                 "plugins": {"twisted_web.py": "import WEBPLUG",
+                             "twisted_words.py": "import WORDPLUG",
+                             "twisted_web2.py": "import WEB2",
+                             "twisted_vfs.py": "import VFS"}},
+            "doc": {"web2": {"excluded!": "yay"},
+                    "vfs": {"unrelated": "whatever"},
+                    "core": {"howto": {"template.tpl": self.template},
+                             "index.xhtml": coreIndexInput}}}
+
+        outStructure = {
+            "README": "Twisted",
+            "unrelated": "x",
+            "LICENSE": "copyright!",
+            "setup.py": "import toplevel",
+            "bin": {"twistd": "TWISTD"},
+            "twisted":
+                {"words": {"__init__.py": "import WORDS"},
+                 "plugins": {"twisted_web.py": "import WEBPLUG",
+                             "twisted_words.py": "import WORDPLUG"}},
+            "doc": {"core": {"howto": {"template.tpl": self.template},
+                             "index.html": coreIndexOutput}}}
+        self.createStructure(self.rootDir, structure)
+
+        outputFile = self.builder.buildTwisted("10.0.0")
+
+        self.assertExtractedStructure(outputFile, outStructure)
+
+
+    def test_subProjectLayout(self):
+        """
+        The subproject tarball includes files like so:
+
+        1. twisted/<subproject>/topfiles defines the files that will be in the
+           top level in the tarball, except LICENSE, which comes from the real
+           top-level directory.
+        2. twisted/<subproject> is included, but without the topfiles entry
+           in that directory. No other twisted subpackages are included.
+        3. twisted/plugins/twisted_<subproject>.py is included, but nothing
+           else in plugins is.
+        """
+        structure = {
+            "README": "HI!@",
+            "unrelated": "x",
+            "LICENSE": "copyright!",
+            "setup.py": "import toplevel",
+            "bin": {"web": {"websetroot": "SET ROOT"},
+                    "words": {"im": "#!im"}},
+            "twisted":
+                {"web":
+                     {"__init__.py": "import WEB",
+                      "topfiles": {"setup.py": "import WEBINSTALL",
+                                   "README": "WEB!"}},
+                 "words": {"__init__.py": "import WORDS"},
+                 "plugins": {"twisted_web.py": "import WEBPLUG",
+                             "twisted_words.py": "import WORDPLUG"}}}
+
+        outStructure = {
+            "README": "WEB!",
+            "LICENSE": "copyright!",
+            "setup.py": "import WEBINSTALL",
+            "bin": {"websetroot": "SET ROOT"},
+            "twisted": {"web": {"__init__.py": "import WEB"},
+                        "plugins": {"twisted_web.py": "import WEBPLUG"}}}
+
+        self.createStructure(self.rootDir, structure)
+
+        outputFile = self.builder.buildSubProject("web", "0.3.0")
+
+        self.assertExtractedStructure(outputFile, outStructure)
+
+
+    def test_minimalSubProjectLayout(self):
+        """
+        L{DistributionBuilder.buildSubProject} works with minimal subprojects.
+        """
+        structure = {
+            "LICENSE": "copyright!",
+            "bin": {},
+            "twisted":
+                {"web": {"__init__.py": "import WEB",
+                         "topfiles": {"setup.py": "import WEBINSTALL"}},
+                 "plugins": {}}}
+
+        outStructure = {
+            "setup.py": "import WEBINSTALL",
+            "LICENSE": "copyright!",
+            "twisted": {"web": {"__init__.py": "import WEB"}}}
+
+        self.createStructure(self.rootDir, structure)
+
+        outputFile = self.builder.buildSubProject("web", "0.3.0")
+
+        self.assertExtractedStructure(outputFile, outStructure)
+
+
+    def test_subProjectDocBuilding(self):
+        """
+        When building a subproject release, documentation should be built with
+        lore.
+        """
+        loreInput, loreOutput = self.getArbitraryLoreInputAndOutput("0.3.0")
+        manInput = self.getArbitraryManInput()
+        manOutput = self.getArbitraryManHTMLOutput("0.3.0", "../howto/")
+        structure = {
+            "LICENSE": "copyright!",
+            "twisted": {"web": {"__init__.py": "import WEB",
+                                "topfiles": {"setup.py": "import WEBINST"}}},
+            "doc": {"web": {"howto": {"index.xhtml": loreInput},
+                            "man": {"twistd.1": manInput}},
+                    "core": {"howto": {"template.tpl": self.template}}
+                    }
+            }
+
+        outStructure = {
+            "LICENSE": "copyright!",
+            "setup.py": "import WEBINST",
+            "twisted": {"web": {"__init__.py": "import WEB"}},
+            "doc": {"howto": {"index.html": loreOutput},
+                    "man": {"twistd.1": manInput,
+                            "twistd-man.html": manOutput}}}
+
+        self.createStructure(self.rootDir, structure)
+
+        outputFile = self.builder.buildSubProject("web", "0.3.0")
+
+        self.assertExtractedStructure(outputFile, outStructure)
+
+
+    def test_coreProjectLayout(self):
+        """
+        The core tarball looks a lot like a subproject tarball, except it
+        doesn't include:
+
+        - Python packages from other subprojects
+        - plugins from other subprojects
+        - scripts from other subprojects
+        """
+        indexInput, indexOutput = self.getArbitraryLoreInputAndOutput(
+            "8.0.0", prefix="howto/")
+        howtoInput, howtoOutput = self.getArbitraryLoreInputAndOutput("8.0.0")
+        specInput, specOutput = self.getArbitraryLoreInputAndOutput(
+            "8.0.0", prefix="../howto/")
+        upgradeInput, upgradeOutput = self.getArbitraryLoreInputAndOutput(
+            "8.0.0", prefix="../howto/")
+        tutorialInput, tutorialOutput = self.getArbitraryLoreInputAndOutput(
+            "8.0.0", prefix="../")
+
+        structure = {
+            "LICENSE": "copyright!",
+            "twisted": {"__init__.py": "twisted",
+                        "python": {"__init__.py": "python",
+                                   "roots.py": "roots!"},
+                        "conch": {"__init__.py": "conch",
+                                  "unrelated.py": "import conch"},
+                        "plugin.py": "plugin",
+                        "plugins": {"twisted_web.py": "webplug",
+                                    "twisted_whatever.py": "include!",
+                                    "cred.py": "include!"},
+                        "topfiles": {"setup.py": "import CORE",
+                                     "README": "core readme"}},
+            "doc": {"core": {"howto": {"template.tpl": self.template,
+                                       "index.xhtml": howtoInput,
+                                       "tutorial":
+                                           {"index.xhtml": tutorialInput}},
+                             "specifications": {"index.xhtml": specInput},
+                             "upgrades": {"index.xhtml": upgradeInput},
+                             "examples": {"foo.py": "foo.py"},
+                             "index.xhtml": indexInput},
+                    "web": {"howto": {"index.xhtml": "webindex"}}},
+            "bin": {"twistd": "TWISTD",
+                    "web": {"websetroot": "websetroot"}}
+            }
+
+        outStructure = {
+            "LICENSE": "copyright!",
+            "setup.py": "import CORE",
+            "README": "core readme",
+            "twisted": {"__init__.py": "twisted",
+                        "python": {"__init__.py": "python",
+                                   "roots.py": "roots!"},
+                        "plugin.py": "plugin",
+                        "plugins": {"twisted_whatever.py": "include!",
+                                    "cred.py": "include!"}},
+            "doc": {"howto": {"template.tpl": self.template,
+                              "index.html": howtoOutput,
+                              "tutorial": {"index.html": tutorialOutput}},
+                    "specifications": {"index.html": specOutput},
+                    "upgrades": {"index.html": upgradeOutput},
+                    "examples": {"foo.py": "foo.py"},
+                    "index.html": indexOutput},
+            "bin": {"twistd": "TWISTD"},
+            }
+
+        self.createStructure(self.rootDir, structure)
+        outputFile = self.builder.buildCore("8.0.0")
+        self.assertExtractedStructure(outputFile, outStructure)
+
+
+    def test_apiBaseURL(self):
+        """
+        L{DistributionBuilder} builds documentation with the specified
+        API base URL.
+        """
+        apiBaseURL = "http://%s"
+        builder = DistributionBuilder(self.rootDir, self.outputDir,
+                                      apiBaseURL=apiBaseURL)
+        loreInput, loreOutput = self.getArbitraryLoreInputAndOutput(
+            "0.3.0", apiBaseURL=apiBaseURL)
+        structure = {
+            "LICENSE": "copyright!",
+            "twisted": {"web": {"__init__.py": "import WEB",
+                                "topfiles": {"setup.py": "import WEBINST"}}},
+            "doc": {"web": {"howto": {"index.xhtml": loreInput}},
+                    "core": {"howto": {"template.tpl": self.template}}
+                    }
+            }
+
+        outStructure = {
+            "LICENSE": "copyright!",
+            "setup.py": "import WEBINST",
+            "twisted": {"web": {"__init__.py": "import WEB"}},
+            "doc": {"howto": {"index.html": loreOutput}}}
+
+        self.createStructure(self.rootDir, structure)
+        outputFile = builder.buildSubProject("web", "0.3.0")
+        self.assertExtractedStructure(outputFile, outStructure)
+
+
+
+class IsDistributableTest(TestCase):
+    """
+    Tests for L{isDistributable}.
+    """
+
+
+    def test_fixedNamesExcluded(self):
+        """
+        L{isDistributable} denies certain fixed names from being packaged.
+        """
+        self.failUnlessEqual(isDistributable(FilePath("foo/_darcs")), False)
+
+
+    def test_hiddenFilesExcluded(self):
+        """
+        L{isDistributable} denies names beginning with a ".".
+        """
+        self.failUnlessEqual(isDistributable(FilePath("foo/.svn")), False)
+
+
+    def test_byteCodeFilesExcluded(self):
+        """
+        L{isDistributable} denies Python bytecode files.
+        """
+        self.failUnlessEqual(isDistributable(FilePath("foo/bar.pyc")), False)
+        self.failUnlessEqual(isDistributable(FilePath("foo/bar.pyo")), False)
+
+
+    def test_otherFilesIncluded(self):
+        """
+        L{isDistributable} allows files with other names.
+        """
+        self.failUnlessEqual(isDistributable(FilePath("foo/bar.py")), True)
+        self.failUnlessEqual(isDistributable(FilePath("foo/README")), True)
+        self.failUnlessEqual(isDistributable(FilePath("foo/twisted")), True)
+
+
+
+class MakeAPIBaseURLTest(TestCase):
+    """
+    Tests for L{makeAPIBaseURL}.
+    """
+
+
+    def test_makeAPIBaseURLIsSubstitutable(self):
+        """
+        L{makeAPIBaseURL} has a place to subtitute an API name.
+        """
+        template = makeAPIBaseURL("12.34")
+
+        # Substitute in an API name.
+        url = template % ("sasquatch",)
+
+        self.assertEqual(url,
+                "http://twistedmatrix.com/documents/12.34/api/sasquatch.html")
+
+
+
+class FilePathDeltaTest(TestCase):
+    """
+    Tests for L{filePathDelta}.
+    """
+
+    def test_filePathDeltaSubdir(self):
+        """
+        L{filePathDelta} can create a simple relative path to a child path.
+        """
+        self.assertEquals(filePathDelta(FilePath("/foo/bar"),
+                                        FilePath("/foo/bar/baz")),
+                          ["baz"])
+
+
+    def test_filePathDeltaSiblingDir(self):
+        """
+        L{filePathDelta} can traverse upwards to create relative paths to
+        siblings.
+        """
+        self.assertEquals(filePathDelta(FilePath("/foo/bar"),
+                                        FilePath("/foo/baz")),
+                          ["..", "baz"])
+
+
+    def test_filePathNoCommonElements(self):
+        """
+        L{filePathDelta} can create relative paths to totally unrelated paths
+        for maximum portability.
+        """
+        self.assertEquals(filePathDelta(FilePath("/foo/bar"),
+                                        FilePath("/baz/quux")),
+                          ["..", "..", "baz", "quux"])
+
+
+    def test_filePathDeltaSimilarEndElements(self):
+        """
+        L{filePathDelta} doesn't take into account final elements when
+        comparing 2 paths, but stops at the first difference.
+        """
+        self.assertEquals(filePathDelta(FilePath("/foo/bar/bar/spam"),
+                                        FilePath("/foo/bar/baz/spam")),
+                          ["..", "..", "baz", "spam"])

twisted/python/test/test_dist.py

@@ -13 +13 @@
 from twisted.trial.unittest import TestCase
 
 from twisted.python import dist
-from twisted.python.dist import get_setup_args, ConditionalExtension
+from twisted.python.dist import (get_setup_args, ConditionalExtension,
+                                 install_data_twisted, build_scripts_twisted,
+                                 getDataFiles)
 from twisted.python.filepath import FilePath
 
 
@@ -25 +27 @@
         """
         Passing C{conditionalExtensions} as a list of L{ConditionalExtension}
         objects to get_setup_args inserts a custom build_ext into the result
-        which knows how to check whether they should be 
+        which knows how to check whether they should be.
         """
         good_ext = ConditionalExtension("whatever", ["whatever.c"],
                                         condition=lambda b: True)
@@ -55 +57 @@
         self.assertEquals(ext.define_macros, [("whatever", 2), ("WIN32", 1)])
 
 
+    def test_defaultCmdClasses(self):
+        """
+        get_setup_args supplies default values for the cmdclass keyword.
+        """
+        args = get_setup_args()
+        self.assertIn('cmdclass', args)
+        cmdclass = args['cmdclass']
+        self.assertIn('install_data', cmdclass)
+        self.assertEquals(cmdclass['install_data'], install_data_twisted)
+        self.assertIn('build_scripts', cmdclass)
+        self.assertEquals(cmdclass['build_scripts'], build_scripts_twisted)
+
+
+    def test_settingCmdClasses(self):
+        """
+        get_setup_args allows new cmdclasses to be added.
+        """
+        args = get_setup_args(cmdclass={'foo': 'bar'})
+        self.assertEquals(args['cmdclass']['foo'], 'bar')
+
+
+    def test_overridingCmdClasses(self):
+        """
+        get_setup_args allows choosing which defaults to override.
+        """
+        args = get_setup_args(cmdclass={'install_data': 'baz'})
+
+        # Overridden cmdclass should be overridden
+        self.assertEquals(args['cmdclass']['install_data'], 'baz')
+
+        # Non-overridden cmdclasses should still be set to defaults.
+        self.assertEquals(args['cmdclass']['build_scripts'],
+                          build_scripts_twisted)
+
+
 
 class GetVersionTest(TestCase):
     """
@@ -171 +208 @@
         os.mkdir(basedir)
         scripts = dist.getScripts('noscripts', basedir=basedir)
         self.assertEquals(scripts, [])
+
+
+
+class GetDataFilesTests(TestCase):
+    """
+    Tests for L{getDataFiles}.
+    """
+
+    def _makeBaseDir(self):
+        """
+        Make a directory for getDataFiles to search.
+        """
+        rawBaseDir = os.path.join(".", self.mktemp())
+        baseDir = FilePath(rawBaseDir)
+        baseDir.makedirs()
+
+        return rawBaseDir, baseDir
+
+
+    def test_basicOperation(self):
+        """
+        L{getDataFiles} finds a single data file in a given directory.
+        """
+        # The directory where we'll put our data file.
+        rawBaseDir, baseDir = self._makeBaseDir()
+
+        # A data file to be found.
+        baseDir.child("foo.txt").touch()
+
+        results = getDataFiles(baseDir.path)
+        self.assertEquals(
+            results,
+            [(rawBaseDir, [os.path.join(rawBaseDir, "foo.txt")])])
+
+
+    def test_directoryRecursion(self):
+        """
+        L{getDataFiles} searches for data files inside subdirectories.
+        """
+        rawBaseDir, baseDir = self._makeBaseDir()
+
+        subDir = baseDir.child("foo")
+        subDir.makedirs()
+
+        subDir.child("bar.txt").touch()
+
+        subSubDir = subDir.child("baz")
+        subSubDir.makedirs()
+
+        subSubDir.child("qux.txt").touch()
+
+        results = getDataFiles(baseDir.path)
+        self.assertEquals(
+            results,
+            [(os.path.join(rawBaseDir, "foo"),
+              [os.path.join(rawBaseDir, "foo", "bar.txt")]),
+             (os.path.join(rawBaseDir, "foo/baz"),
+              [os.path.join(rawBaseDir, "foo/baz", "qux.txt")])])
+
+
+    def test_ignoreVCSMetadata(self):
+        """
+        L{getDataFiles} ignores Subversion metadata files.
+        """
+        rawBaseDir, baseDir = self._makeBaseDir()
+
+        # Top-level directory contains a VCS dir, containing ignorable data.
+        vcsDir = baseDir.child(".svn")
+        vcsDir.makedirs()
+        vcsDir.child("data.txt").touch()
+
+        # Subdirectory contains a valid data file.
+        subDir = baseDir.child("foo")
+        subDir.makedirs()
+        subDir.child("bar.txt").touch()
+
+        # Subdirectory contains another VCS dir, with more ignorable data.
+        subVcsDir = subDir.child("_darcs")
+        subVcsDir.makedirs()
+        subVcsDir.child("data.txt").touch()
+
+        # Subdirectory contains an ignorable VCS file.
+        subDir.child(".cvsignore").touch()
+
+        results = getDataFiles(baseDir.path)
+        self.assertEquals(
+            results,
+            [(os.path.join(rawBaseDir, "foo"),
+              [os.path.join(rawBaseDir, "foo", "bar.txt")])])
+
+
+    def test_ignoreArbitrarySubdirectories(self):
+        """
+        L{getDataFiles} ignores any filenames it's asked to ignore.
+        """
+        rawBaseDir, baseDir = self._makeBaseDir()
+
+        subDir = baseDir.child("foo")
+        subDir.makedirs()
+
+        # Make an ordinary subdirectory with some data files.
+        subDir.child("bar.txt").touch()
+        subDir.child("ignorable").touch() # not a dir, won't be ignored
+
+        # Make a subdirectory with an ignorable name, and some data files.
+        ignorableSubDir = baseDir.child("ignorable")
+        ignorableSubDir.makedirs()
+        ignorableSubDir.child("bar.txt").touch()
+
+        results = getDataFiles(baseDir.path, ignore=["ignorable"])
+        self.assertEquals(
+            results,
+            [(os.path.join(rawBaseDir, "foo"),
+              [os.path.join(rawBaseDir, "foo", "bar.txt"),
+               os.path.join(rawBaseDir, "foo", "ignorable")])])
+
+
+    def test_ignoreNonDataFiles(self):
+        """
+        L{getDataFiles} ignores Python code, backup files and bytecode.
+        """
+        rawBaseDir, baseDir = self._makeBaseDir()
+
+        # All these are not data files, and should be ignored.
+        baseDir.child("module.py").touch()
+        baseDir.child("module.pyc").touch()
+        baseDir.child("module.pyo").touch()
+
+        subDir = baseDir.child("foo")
+        subDir.makedirs()
+
+        subDir.child("bar.txt").touch()
+
+        # An editor-made backup of bar.txt should be ignored.
+        subDir.child("bar.txt~").touch()
+
+        results = getDataFiles(baseDir.path)
+        self.assertEquals(
+            results,
+            [(os.path.join(rawBaseDir, "foo"),
+              [os.path.join(rawBaseDir, "foo", "bar.txt")])])
+
+
+    def test_pathsRelativeToParent(self):
+        """
+        L{getDataFiles} returns paths relative to the parent parameter.
+        """
+        rawBaseDir, baseDir = self._makeBaseDir()
+
+        # munge rawBaseDir in a way that we can recognise later.
+        mungedBaseDir = os.path.join(rawBaseDir, "foo/../")
+
+        subDir = baseDir.child("foo")
+        subDir.makedirs()
+
+        subDir.child("bar.txt").touch()
+
+        results = getDataFiles(subDir.path, parent=mungedBaseDir)
+        self.assertEquals(
+            results,
+            [(os.path.join(mungedBaseDir, "foo"),
+              [os.path.join(mungedBaseDir, "foo", "bar.txt")])])

twisted/python/test/test_release.py

@@ -13 +13 @@
 import operator
 import os, sys, signal
 from StringIO import StringIO
-import tarfile
-from xml.dom import minidom as dom
 
 from datetime import date
 
@@ -31 +29 @@
 from twisted.python._release import updateTwistedVersionInformation, Project
 from twisted.python._release import generateVersionFileData
 from twisted.python._release import changeAllProjectVersions
-from twisted.python._release import VERSION_OFFSET, DocBuilder, ManBuilder
-from twisted.python._release import NoDocumentsFound, filePathDelta
+from twisted.python._release import VERSION_OFFSET
 from twisted.python._release import CommandFailed, BookBuilder
-from twisted.python._release import DistributionBuilder, APIBuilder
+from twisted.python._release import APIBuilder
 from twisted.python._release import BuildAPIDocsScript
 from twisted.python._release import buildAllTarballs, runCommand
 from twisted.python._release import UncleanWorkingDirectory, NotWorkingDirectory
 from twisted.python._release import ChangeVersionsScript, BuildTarballsScript
 from twisted.python._release import NewsBuilder
 
+from twisted.python.test.test__dist import loreSkip, StructureAssertingMixin
+from twisted.python.test.test__dist import BuilderTestsMixin
+from twisted.python.test.test__dist import DistributionBuilderTestBase
+
 if os.name != 'posix':
     skip = "Release toolchain only supported on POSIX."
 else:
@@ -57 +58 @@
 
 
 try:
+    from popen2 import Popen4
+except ImportError:
+    popen4Skip = "popen2.Popen4 is not available."
+else:
+    popen4Skip = skip
+
+try:
     import pydoctor.driver
     # it might not be installed, or it might use syntax not available in
     # this version of Python.
@@ -92 +100 @@
 
 
 
-class StructureAssertingMixin(object):
-    """
-    A mixin for L{TestCase} subclasses which provides some methods for asserting
-    the structure and contents of directories and files on the filesystem.
-    """
-    def createStructure(self, root, dirDict):
-        """
-        Create a set of directories and files given a dict defining their
-        structure.
-
-        @param root: The directory in which to create the structure.  It must
-            already exist.
-        @type root: L{FilePath}
-
-        @param dirDict: The dict defining the structure. Keys should be strings
-            naming files, values should be strings describing file contents OR
-            dicts describing subdirectories.  All files are written in binary
-            mode.  Any string values are assumed to describe text files and
-            will have their newlines replaced with the platform-native newline
-            convention.  For example::
-
-                {"foofile": "foocontents",
-                 "bardir": {"barfile": "bar\ncontents"}}
-        @type dirDict: C{dict}
-        """
-        for x in dirDict:
-            child = root.child(x)
-            if isinstance(dirDict[x], dict):
-                child.createDirectory()
-                self.createStructure(child, dirDict[x])
-            else:
-                child.setContent(dirDict[x].replace('\n', os.linesep))
-
-    def assertStructure(self, root, dirDict):
-        """
-        Assert that a directory is equivalent to one described by a dict.
-
-        @param root: The filesystem directory to compare.
-        @type root: L{FilePath}
-        @param dirDict: The dict that should describe the contents of the
-            directory. It should be the same structure as the C{dirDict}
-            parameter to L{createStructure}.
-        @type dirDict: C{dict}
-        """
-        children = [x.basename() for x in root.children()]
-        for x in dirDict:
-            child = root.child(x)
-            if isinstance(dirDict[x], dict):
-                self.assertTrue(child.isdir(), "%s is not a dir!"
-                                % (child.path,))
-                self.assertStructure(child, dirDict[x])
-            else:
-                a = child.getContent().replace(os.linesep, '\n')
-                self.assertEquals(a, dirDict[x], child.path)
-            children.remove(x)
-        if children:
-            self.fail("There were extra children in %s: %s"
-                      % (root.path, children))
-
-
-    def assertExtractedStructure(self, outputFile, dirDict):
-        """
-        Assert that a tarfile content is equivalent to one described by a dict.
-
-        @param outputFile: The tar file built by L{DistributionBuilder}.
-        @type outputFile: L{FilePath}.
-        @param dirDict: The dict that should describe the contents of the
-            directory. It should be the same structure as the C{dirDict}
-            parameter to L{createStructure}.
-        @type dirDict: C{dict}
-        """
-        tarFile = tarfile.TarFile.open(outputFile.path, "r:bz2")
-        extracted = FilePath(self.mktemp())
-        extracted.createDirectory()
-        for info in tarFile:
-            tarFile.extract(info, path=extracted.path)
-        self.assertStructure(extracted.children()[0], dirDict)
-
-
-
 class ChangeVersionTest(TestCase, StructureAssertingMixin):
     """
     Twisted has the ability to change versions.
@@ -480 +408 @@
 
 
 
-class BuilderTestsMixin(object):
-    """
-    A mixin class which provides various methods for creating sample Lore input
-    and output.
-
-    @cvar template: The lore template that will be used to prepare sample
-    output.
-    @type template: C{str}
-
-    @ivar docCounter: A counter which is incremented every time input is
-        generated and which is included in the documents.
-    @type docCounter: C{int}
-    """
-    template = '''
-    <html>
-    <head><title>Yo:</title></head>
-    <body>
-    <div class="body" />
-    <a href="index.html">Index</a>
-    <span class="version">Version: </span>
-    </body>
-    </html>
-    '''
-
-    def setUp(self):
-        """
-        Initialize the doc counter which ensures documents are unique.
-        """
-        self.docCounter = 0
-
-
-    def assertXMLEqual(self, first, second):
-        """
-        Verify that two strings represent the same XML document.
-        """
-        self.assertEqual(
-            dom.parseString(first).toxml(),
-            dom.parseString(second).toxml())
-
-
-    def getArbitraryOutput(self, version, counter, prefix="", apiBaseURL="%s"):
-        """
-        Get the correct HTML output for the arbitrary input returned by
-        L{getArbitraryLoreInput} for the given parameters.
-
-        @param version: The version string to include in the output.
-        @type version: C{str}
-        @param counter: A counter to include in the output.
-        @type counter: C{int}
-        """
-        document = """\
-<?xml version="1.0"?><html>
-    <head><title>Yo:Hi! Title: %(count)d</title></head>
-    <body>
-    <div class="content">Hi! %(count)d<div class="API"><a href="%(foobarLink)s" title="foobar">foobar</a></div></div>
-    <a href="%(prefix)sindex.html">Index</a>
-    <span class="version">Version: %(version)s</span>
-    </body>
-    </html>"""
-        # Try to normalize irrelevant whitespace.
-        return dom.parseString(
-            document % {"count": counter, "prefix": prefix,
-                        "version": version,
-                        "foobarLink": apiBaseURL % ("foobar",)}).toxml('utf-8')
-
-
-    def getArbitraryLoreInput(self, counter):
-        """
-        Get an arbitrary, unique (for this test case) string of lore input.
-
-        @param counter: A counter to include in the input.
-        @type counter: C{int}
-        """
-        template = (
-            '<html>'
-            '<head><title>Hi! Title: %(count)s</title></head>'
-            '<body>'
-            'Hi! %(count)s'
-            '<div class="API">foobar</div>'
-            '</body>'
-            '</html>')
-        return template % {"count": counter}
-
-
-    def getArbitraryLoreInputAndOutput(self, version, prefix="",
-                                       apiBaseURL="%s"):
-        """
-        Get an input document along with expected output for lore run on that
-        output document, assuming an appropriately-specified C{self.template}.
-
-        @param version: A version string to include in the input and output.
-        @type version: C{str}
-        @param prefix: The prefix to include in the link to the index.
-        @type prefix: C{str}
-
-        @return: A two-tuple of input and expected output.
-        @rtype: C{(str, str)}.
-        """
-        self.docCounter += 1
-        return (self.getArbitraryLoreInput(self.docCounter),
-                self.getArbitraryOutput(version, self.docCounter,
-                                        prefix=prefix, apiBaseURL=apiBaseURL))
-
-
-    def getArbitraryManInput(self):
-        """
-        Get an arbitrary man page content.
-        """
-        return """.TH MANHOLE "1" "August 2001" "" ""
-.SH NAME
-manhole \- Connect to a Twisted Manhole service
-.SH SYNOPSIS
-.B manhole
-.SH DESCRIPTION
-manhole is a GTK interface to Twisted Manhole services. You can execute python
-code as if at an interactive Python console inside a running Twisted process
-with this."""
-
-
-    def getArbitraryManLoreOutput(self):
-        """
-        Get an arbitrary lore input document which represents man-to-lore
-        output based on the man page returned from L{getArbitraryManInput}
-        """
-        return """\
-<?xml version="1.0"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
-    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<html><head>
-<title>MANHOLE.1</title></head>
-<body>
-
-<h1>MANHOLE.1</h1>
-
-<h2>NAME</h2>
-
-<p>manhole - Connect to a Twisted Manhole service
-</p>
-
-<h2>SYNOPSIS</h2>
-
-<p><strong>manhole</strong> </p>
-
-<h2>DESCRIPTION</h2>
-
-<p>manhole is a GTK interface to Twisted Manhole services. You can execute python
-code as if at an interactive Python console inside a running Twisted process
-with this.</p>
-
-</body>
-</html>
-"""
-
-    def getArbitraryManHTMLOutput(self, version, prefix=""):
-        """
-        Get an arbitrary lore output document which represents the lore HTML
-        output based on the input document returned from
-        L{getArbitraryManLoreOutput}.
-
-        @param version: A version string to include in the document.
-        @type version: C{str}
-        @param prefix: The prefix to include in the link to the index.
-        @type prefix: C{str}
-        """
-        # Try to normalize the XML a little bit.
-        return dom.parseString("""\
-<?xml version="1.0" ?><html>
-    <head><title>Yo:MANHOLE.1</title></head>
-    <body>
-    <div class="content">
-
-<span/>
-
-<h2>NAME<a name="auto0"/></h2>
-
-<p>manhole - Connect to a Twisted Manhole service
-</p>
-
-<h2>SYNOPSIS<a name="auto1"/></h2>
-
-<p><strong>manhole</strong> </p>
-
-<h2>DESCRIPTION<a name="auto2"/></h2>
-
-<p>manhole is a GTK interface to Twisted Manhole services. You can execute python
-code as if at an interactive Python console inside a running Twisted process
-with this.</p>
-
-</div>
-    <a href="%(prefix)sindex.html">Index</a>
-    <span class="version">Version: %(version)s</span>
-    </body>
-    </html>""" % {
-            'prefix': prefix, 'version': version}).toxml("utf-8")
-
-
-
-class DocBuilderTestCase(TestCase, BuilderTestsMixin):
-    """
-    Tests for L{DocBuilder}.
-
-    Note for future maintainers: The exact byte equality assertions throughout
-    this suite may need to be updated due to minor differences in lore. They
-    should not be taken to mean that Lore must maintain the same byte format
-    forever. Feel free to update the tests when Lore changes, but please be
-    careful.
-    """
-    skip = loreSkip
-
-    def setUp(self):
-        """
-        Set up a few instance variables that will be useful.
-
-        @ivar builder: A plain L{DocBuilder}.
-        @ivar docCounter: An integer to be used as a counter by the
-            C{getArbitrary...} methods.
-        @ivar howtoDir: A L{FilePath} representing a directory to be used for
-            containing Lore documents.
-        @ivar templateFile: A L{FilePath} representing a file with
-            C{self.template} as its content.
-        """
-        BuilderTestsMixin.setUp(self)
-        self.builder = DocBuilder()
-        self.howtoDir = FilePath(self.mktemp())
-        self.howtoDir.createDirectory()
-        self.templateFile = self.howtoDir.child("template.tpl")
-        self.templateFile.setContent(self.template)
-
-
-    def test_build(self):
-        """
-        The L{DocBuilder} runs lore on all .xhtml files within a directory.
-        """
-        version = "1.2.3"
-        input1, output1 = self.getArbitraryLoreInputAndOutput(version)
-        input2, output2 = self.getArbitraryLoreInputAndOutput(version)
-
-        self.howtoDir.child("one.xhtml").setContent(input1)
-        self.howtoDir.child("two.xhtml").setContent(input2)
-
-        self.builder.build(version, self.howtoDir, self.howtoDir,
-                           self.templateFile)
-        out1 = self.howtoDir.child('one.html')
-        out2 = self.howtoDir.child('two.html')
-        self.assertXMLEqual(out1.getContent(), output1)
-        self.assertXMLEqual(out2.getContent(), output2)
-
-
-    def test_noDocumentsFound(self):
-        """
-        The C{build} method raises L{NoDocumentsFound} if there are no
-        .xhtml files in the given directory.
-        """
-        self.assertRaises(
-            NoDocumentsFound,
-            self.builder.build, "1.2.3", self.howtoDir, self.howtoDir,
-            self.templateFile)
-
-
-    def test_parentDocumentLinking(self):
-        """
-        The L{DocBuilder} generates correct links from documents to
-        template-generated links like stylesheets and index backreferences.
-        """
-        input = self.getArbitraryLoreInput(0)
-        tutoDir = self.howtoDir.child("tutorial")
-        tutoDir.createDirectory()
-        tutoDir.child("child.xhtml").setContent(input)
-        self.builder.build("1.2.3", self.howtoDir, tutoDir, self.templateFile)
-        outFile = tutoDir.child('child.html')
-        self.assertIn('<a href="../index.html">Index</a>',
-                      outFile.getContent())
-
-
-    def test_siblingDirectoryDocumentLinking(self):
-        """
-        It is necessary to generate documentation in a directory foo/bar where
-        stylesheet and indexes are located in foo/baz. Such resources should be
-        appropriately linked to.
-        """
-        input = self.getArbitraryLoreInput(0)
-        resourceDir = self.howtoDir.child("resources")
-        docDir = self.howtoDir.child("docs")
-        docDir.createDirectory()
-        docDir.child("child.xhtml").setContent(input)
-        self.builder.build("1.2.3", resourceDir, docDir, self.templateFile)
-        outFile = docDir.child('child.html')
-        self.assertIn('<a href="../resources/index.html">Index</a>',
-                      outFile.getContent())
-
-
-    def test_apiLinking(self):
-        """
-        The L{DocBuilder} generates correct links from documents to API
-        documentation.
-        """
-        version = "1.2.3"
-        input, output = self.getArbitraryLoreInputAndOutput(version)
-        self.howtoDir.child("one.xhtml").setContent(input)
-
-        self.builder.build(version, self.howtoDir, self.howtoDir,
-                           self.templateFile, "scheme:apilinks/%s.ext")
-        out = self.howtoDir.child('one.html')
-        self.assertIn(
-            '<a href="scheme:apilinks/foobar.ext" title="foobar">foobar</a>',
-            out.getContent())
-
-
-    def test_deleteInput(self):
-        """
-        L{DocBuilder.build} can be instructed to delete the input files after
-        generating the output based on them.
-        """
-        input1 = self.getArbitraryLoreInput(0)
-        self.howtoDir.child("one.xhtml").setContent(input1)
-        self.builder.build("whatever", self.howtoDir, self.howtoDir,
-                           self.templateFile, deleteInput=True)
-        self.assertTrue(self.howtoDir.child('one.html').exists())
-        self.assertFalse(self.howtoDir.child('one.xhtml').exists())
-
-
-    def test_doNotDeleteInput(self):
-        """
-        Input will not be deleted by default.
-        """
-        input1 = self.getArbitraryLoreInput(0)
-        self.howtoDir.child("one.xhtml").setContent(input1)
-        self.builder.build("whatever", self.howtoDir, self.howtoDir,
-                           self.templateFile)
-        self.assertTrue(self.howtoDir.child('one.html').exists())
-        self.assertTrue(self.howtoDir.child('one.xhtml').exists())
-
-
-    def test_getLinkrelToSameDirectory(self):
-        """
-        If the doc and resource directories are the same, the linkrel should be
-        an empty string.
-        """
-        linkrel = self.builder.getLinkrel(FilePath("/foo/bar"),
-                                          FilePath("/foo/bar"))
-        self.assertEquals(linkrel, "")
-
-
-    def test_getLinkrelToParentDirectory(self):
-        """
-        If the doc directory is a child of the resource directory, the linkrel
-        should make use of '..'.
-        """
-        linkrel = self.builder.getLinkrel(FilePath("/foo"),
-                                          FilePath("/foo/bar"))
-        self.assertEquals(linkrel, "../")
-
-
-    def test_getLinkrelToSibling(self):
-        """
-        If the doc directory is a sibling of the resource directory, the
-        linkrel should make use of '..' and a named segment.
-        """
-        linkrel = self.builder.getLinkrel(FilePath("/foo/howto"),
-                                          FilePath("/foo/examples"))
-        self.assertEquals(linkrel, "../howto/")
-
-
-    def test_getLinkrelToUncle(self):
-        """
-        If the doc directory is a sibling of the parent of the resource
-        directory, the linkrel should make use of multiple '..'s and a named
-        segment.
-        """
-        linkrel = self.builder.getLinkrel(FilePath("/foo/howto"),
-                                          FilePath("/foo/examples/quotes"))
-        self.assertEquals(linkrel, "../../howto/")
-
-
-
 class APIBuilderTestCase(TestCase):
     """
     Tests for L{APIBuilder}.
@@ -994 +547 @@
 
 
 
-class ManBuilderTestCase(TestCase, BuilderTestsMixin):
-    """
-    Tests for L{ManBuilder}.
-    """
-    skip = loreSkip
-
-    def setUp(self):
-        """
-        Set up a few instance variables that will be useful.
-
-        @ivar builder: A plain L{ManBuilder}.
-        @ivar manDir: A L{FilePath} representing a directory to be used for
-            containing man pages.
-        """
-        BuilderTestsMixin.setUp(self)
-        self.builder = ManBuilder()
-        self.manDir = FilePath(self.mktemp())
-        self.manDir.createDirectory()
-
-
-    def test_noDocumentsFound(self):
-        """
-        L{ManBuilder.build} raises L{NoDocumentsFound} if there are no
-        .1 files in the given directory.
-        """
-        self.assertRaises(NoDocumentsFound, self.builder.build, self.manDir)
-
-
-    def test_build(self):
-        """
-        Check that L{ManBuilder.build} find the man page in the directory, and
-        successfully produce a Lore content.
-        """
-        manContent = self.getArbitraryManInput()
-        self.manDir.child('test1.1').setContent(manContent)
-        self.builder.build(self.manDir)
-        output = self.manDir.child('test1-man.xhtml').getContent()
-        expected = self.getArbitraryManLoreOutput()
-        # No-op on *nix, fix for windows
-        expected = expected.replace('\n', os.linesep)
-        self.assertEquals(output, expected)
-
-
-    def test_toHTML(self):
-        """
-        Check that the content output by C{build} is compatible as input of
-        L{DocBuilder.build}.
-        """
-        manContent = self.getArbitraryManInput()
-        self.manDir.child('test1.1').setContent(manContent)
-        self.builder.build(self.manDir)
-
-        templateFile = self.manDir.child("template.tpl")
-        templateFile.setContent(DocBuilderTestCase.template)
-        docBuilder = DocBuilder()
-        docBuilder.build("1.2.3", self.manDir, self.manDir,
-                         templateFile)
-        output = self.manDir.child('test1-man.html').getContent()
-
-        self.assertXMLEqual(
-            output,
-            """\
-<?xml version="1.0" ?><html>
-    <head><title>Yo:MANHOLE.1</title></head>
-    <body>
-    <div class="content">
-
-<span/>
-
-<h2>NAME<a name="auto0"/></h2>
-
-<p>manhole - Connect to a Twisted Manhole service
-</p>
-
-<h2>SYNOPSIS<a name="auto1"/></h2>
-
-<p><strong>manhole</strong> </p>
-
-<h2>DESCRIPTION<a name="auto2"/></h2>
-
-<p>manhole is a GTK interface to Twisted Manhole services. You can execute python
-code as if at an interactive Python console inside a running Twisted process
-with this.</p>
-
-</div>
-    <a href="index.html">Index</a>
-    <span class="version">Version: 1.2.3</span>
-    </body>
-    </html>""")
-
-
-
 class BookBuilderTests(TestCase, BuilderTestsMixin):
     """
     Tests for L{BookBuilder}.
@@ -1418 +879 @@
 
 
 
-class FilePathDeltaTest(TestCase):
-    """
-    Tests for L{filePathDelta}.
-    """
-
-    def test_filePathDeltaSubdir(self):
-        """
-        L{filePathDelta} can create a simple relative path to a child path.
-        """
-        self.assertEquals(filePathDelta(FilePath("/foo/bar"),
-                                        FilePath("/foo/bar/baz")),
-                          ["baz"])
-
-
-    def test_filePathDeltaSiblingDir(self):
-        """
-        L{filePathDelta} can traverse upwards to create relative paths to
-        siblings.
-        """
-        self.assertEquals(filePathDelta(FilePath("/foo/bar"),
-                                        FilePath("/foo/baz")),
-                          ["..", "baz"])
-
-
-    def test_filePathNoCommonElements(self):
-        """
-        L{filePathDelta} can create relative paths to totally unrelated paths
-        for maximum portability.
-        """
-        self.assertEquals(filePathDelta(FilePath("/foo/bar"),
-                                        FilePath("/baz/quux")),
-                          ["..", "..", "baz", "quux"])
-
-
-    def test_filePathDeltaSimilarEndElements(self):
-        """
-        L{filePathDelta} doesn't take into account final elements when
-        comparing 2 paths, but stops at the first difference.
-        """
-        self.assertEquals(filePathDelta(FilePath("/foo/bar/bar/spam"),
-                                        FilePath("/foo/bar/baz/spam")),
-                          ["..", "..", "baz", "spam"])
-
-
-
 class NewsBuilderTests(TestCase, StructureAssertingMixin):
     """
     Tests for L{NewsBuilder}.
@@ -1869 +1285 @@
 
 
 
-class DistributionBuilderTestBase(BuilderTestsMixin, StructureAssertingMixin,
-                                   TestCase):
-    """
-    Base for tests of L{DistributionBuilder}.
-    """
-    skip = loreSkip
-
-    def setUp(self):
-        BuilderTestsMixin.setUp(self)
-
-        self.rootDir = FilePath(self.mktemp())
-        self.rootDir.createDirectory()
-
-        self.outputDir = FilePath(self.mktemp())
-        self.outputDir.createDirectory()
-        self.builder = DistributionBuilder(self.rootDir, self.outputDir)
-
-
-
-class DistributionBuilderTest(DistributionBuilderTestBase):
-
-    def test_twistedDistribution(self):
-        """
-        The Twisted tarball contains everything in the source checkout, with
-        built documentation.
-        """
-        loreInput, loreOutput = self.getArbitraryLoreInputAndOutput("10.0.0")
-        manInput1 = self.getArbitraryManInput()
-        manOutput1 = self.getArbitraryManHTMLOutput("10.0.0", "../howto/")
-        manInput2 = self.getArbitraryManInput()
-        manOutput2 = self.getArbitraryManHTMLOutput("10.0.0", "../howto/")
-        coreIndexInput, coreIndexOutput = self.getArbitraryLoreInputAndOutput(
-            "10.0.0", prefix="howto/")
-
-        structure = {
-            "README": "Twisted",
-            "unrelated": "x",
-            "LICENSE": "copyright!",
-            "setup.py": "import toplevel",
-            "bin": {"web": {"websetroot": "SET ROOT"},
-                    "twistd": "TWISTD"},
-            "twisted":
-                {"web":
-                     {"__init__.py": "import WEB",
-                      "topfiles": {"setup.py": "import WEBINSTALL",
-                                   "README": "WEB!"}},
-                 "words": {"__init__.py": "import WORDS"},
-                 "plugins": {"twisted_web.py": "import WEBPLUG",
-                             "twisted_words.py": "import WORDPLUG"}},
-            "doc": {"web": {"howto": {"index.xhtml": loreInput},
-                            "man": {"websetroot.1": manInput2}},
-                    "core": {"howto": {"template.tpl": self.template},
-                             "man": {"twistd.1": manInput1},
-                             "index.xhtml": coreIndexInput}}}
-
-        outStructure = {
-            "README": "Twisted",
-            "unrelated": "x",
-            "LICENSE": "copyright!",
-            "setup.py": "import toplevel",
-            "bin": {"web": {"websetroot": "SET ROOT"},
-                    "twistd": "TWISTD"},
-            "twisted":
-                {"web": {"__init__.py": "import WEB",
-                         "topfiles": {"setup.py": "import WEBINSTALL",
-                                      "README": "WEB!"}},
-                 "words": {"__init__.py": "import WORDS"},
-                 "plugins": {"twisted_web.py": "import WEBPLUG",
-                             "twisted_words.py": "import WORDPLUG"}},
-            "doc": {"web": {"howto": {"index.html": loreOutput},
-                            "man": {"websetroot.1": manInput2,
-                                    "websetroot-man.html": manOutput2}},
-                    "core": {"howto": {"template.tpl": self.template},
-                             "man": {"twistd.1": manInput1,
-                                     "twistd-man.html": manOutput1},
-                             "index.html": coreIndexOutput}}}
-
-        self.createStructure(self.rootDir, structure)
-
-        outputFile = self.builder.buildTwisted("10.0.0")
-
-        self.assertExtractedStructure(outputFile, outStructure)
-
-
-    def test_twistedDistributionExcludesWeb2AndVFSAndAdmin(self):
-        """
-        The main Twisted distribution does not include web2 or vfs, or the
-        bin/admin directory.
-        """
-        loreInput, loreOutput = self.getArbitraryLoreInputAndOutput("10.0.0")
-        coreIndexInput, coreIndexOutput = self.getArbitraryLoreInputAndOutput(
-            "10.0.0", prefix="howto/")
-
-        structure = {
-            "README": "Twisted",
-            "unrelated": "x",
-            "LICENSE": "copyright!",
-            "setup.py": "import toplevel",
-            "bin": {"web2": {"websetroot": "SET ROOT"},
-                    "vfs": {"vfsitup": "hee hee"},
-                    "twistd": "TWISTD",
-                    "admin": {"build-a-thing": "yay"}},
-            "twisted":
-                {"web2":
-                     {"__init__.py": "import WEB",
-                      "topfiles": {"setup.py": "import WEBINSTALL",
-                                   "README": "WEB!"}},
-                 "vfs":
-                     {"__init__.py": "import VFS",
-                      "blah blah": "blah blah"},
-                 "words": {"__init__.py": "import WORDS"},
-                 "plugins": {"twisted_web.py": "import WEBPLUG",
-                             "twisted_words.py": "import WORDPLUG",
-                             "twisted_web2.py": "import WEB2",
-                             "twisted_vfs.py": "import VFS"}},
-            "doc": {"web2": {"excluded!": "yay"},
-                    "vfs": {"unrelated": "whatever"},
-                    "core": {"howto": {"template.tpl": self.template},
-                             "index.xhtml": coreIndexInput}}}
-
-        outStructure = {
-            "README": "Twisted",
-            "unrelated": "x",
-            "LICENSE": "copyright!",
-            "setup.py": "import toplevel",
-            "bin": {"twistd": "TWISTD"},
-            "twisted":
-                {"words": {"__init__.py": "import WORDS"},
-                 "plugins": {"twisted_web.py": "import WEBPLUG",
-                             "twisted_words.py": "import WORDPLUG"}},
-            "doc": {"core": {"howto": {"template.tpl": self.template},
-                             "index.html": coreIndexOutput}}}
-        self.createStructure(self.rootDir, structure)
-
-        outputFile = self.builder.buildTwisted("10.0.0")
-
-        self.assertExtractedStructure(outputFile, outStructure)
-
-
-    def test_subProjectLayout(self):
-        """
-        The subproject tarball includes files like so:
-
-        1. twisted/<subproject>/topfiles defines the files that will be in the
-           top level in the tarball, except LICENSE, which comes from the real
-           top-level directory.
-        2. twisted/<subproject> is included, but without the topfiles entry
-           in that directory. No other twisted subpackages are included.
-        3. twisted/plugins/twisted_<subproject>.py is included, but nothing
-           else in plugins is.
-        """
-        structure = {
-            "README": "HI!@",
-            "unrelated": "x",
-            "LICENSE": "copyright!",
-            "setup.py": "import toplevel",
-            "bin": {"web": {"websetroot": "SET ROOT"},
-                    "words": {"im": "#!im"}},
-            "twisted":
-                {"web":
-                     {"__init__.py": "import WEB",
-                      "topfiles": {"setup.py": "import WEBINSTALL",
-                                   "README": "WEB!"}},
-                 "words": {"__init__.py": "import WORDS"},
-                 "plugins": {"twisted_web.py": "import WEBPLUG",
-                             "twisted_words.py": "import WORDPLUG"}}}
-
-        outStructure = {
-            "README": "WEB!",
-            "LICENSE": "copyright!",
-            "setup.py": "import WEBINSTALL",
-            "bin": {"websetroot": "SET ROOT"},
-            "twisted": {"web": {"__init__.py": "import WEB"},
-                        "plugins": {"twisted_web.py": "import WEBPLUG"}}}
-
-        self.createStructure(self.rootDir, structure)
-
-        outputFile = self.builder.buildSubProject("web", "0.3.0")
-
-        self.assertExtractedStructure(outputFile, outStructure)
-
-
-    def test_minimalSubProjectLayout(self):
-        """
-        buildSubProject should work with minimal subprojects.
-        """
-        structure = {
-            "LICENSE": "copyright!",
-            "bin": {},
-            "twisted":
-                {"web": {"__init__.py": "import WEB",
-                         "topfiles": {"setup.py": "import WEBINSTALL"}},
-                 "plugins": {}}}
-
-        outStructure = {
-            "setup.py": "import WEBINSTALL",
-            "LICENSE": "copyright!",
-            "twisted": {"web": {"__init__.py": "import WEB"}}}
-
-        self.createStructure(self.rootDir, structure)
-
-        outputFile = self.builder.buildSubProject("web", "0.3.0")
-
-        self.assertExtractedStructure(outputFile, outStructure)
-
-
-    def test_subProjectDocBuilding(self):
-        """
-        When building a subproject release, documentation should be built with
-        lore.
-        """
-        loreInput, loreOutput = self.getArbitraryLoreInputAndOutput("0.3.0")
-        manInput = self.getArbitraryManInput()
-        manOutput = self.getArbitraryManHTMLOutput("0.3.0", "../howto/")
-        structure = {
-            "LICENSE": "copyright!",
-            "twisted": {"web": {"__init__.py": "import WEB",
-                                "topfiles": {"setup.py": "import WEBINST"}}},
-            "doc": {"web": {"howto": {"index.xhtml": loreInput},
-                            "man": {"twistd.1": manInput}},
-                    "core": {"howto": {"template.tpl": self.template}}
-                    }
-            }
-
-        outStructure = {
-            "LICENSE": "copyright!",
-            "setup.py": "import WEBINST",
-            "twisted": {"web": {"__init__.py": "import WEB"}},
-            "doc": {"howto": {"index.html": loreOutput},
-                    "man": {"twistd.1": manInput,
-                            "twistd-man.html": manOutput}}}
-
-        self.createStructure(self.rootDir, structure)
-
-        outputFile = self.builder.buildSubProject("web", "0.3.0")
-
-        self.assertExtractedStructure(outputFile, outStructure)
-
-
-    def test_coreProjectLayout(self):
-        """
-        The core tarball looks a lot like a subproject tarball, except it
-        doesn't include:
-
-        - Python packages from other subprojects
-        - plugins from other subprojects
-        - scripts from other subprojects
-        """
-        indexInput, indexOutput = self.getArbitraryLoreInputAndOutput(
-            "8.0.0", prefix="howto/")
-        howtoInput, howtoOutput = self.getArbitraryLoreInputAndOutput("8.0.0")
-        specInput, specOutput = self.getArbitraryLoreInputAndOutput(
-            "8.0.0", prefix="../howto/")
-        upgradeInput, upgradeOutput = self.getArbitraryLoreInputAndOutput(
-            "8.0.0", prefix="../howto/")
-        tutorialInput, tutorialOutput = self.getArbitraryLoreInputAndOutput(
-            "8.0.0", prefix="../")
-
-        structure = {
-            "LICENSE": "copyright!",
-            "twisted": {"__init__.py": "twisted",
-                        "python": {"__init__.py": "python",
-                                   "roots.py": "roots!"},
-                        "conch": {"__init__.py": "conch",
-                                  "unrelated.py": "import conch"},
-                        "plugin.py": "plugin",
-                        "plugins": {"twisted_web.py": "webplug",
-                                    "twisted_whatever.py": "include!",
-                                    "cred.py": "include!"},
-                        "topfiles": {"setup.py": "import CORE",
-                                     "README": "core readme"}},
-            "doc": {"core": {"howto": {"template.tpl": self.template,
-                                       "index.xhtml": howtoInput,
-                                       "tutorial":
-                                           {"index.xhtml": tutorialInput}},
-                             "specifications": {"index.xhtml": specInput},
-                             "upgrades": {"index.xhtml": upgradeInput},
-                             "examples": {"foo.py": "foo.py"},
-                             "index.xhtml": indexInput},
-                    "web": {"howto": {"index.xhtml": "webindex"}}},
-            "bin": {"twistd": "TWISTD",
-                    "web": {"websetroot": "websetroot"}}
-            }
-
-        outStructure = {
-            "LICENSE": "copyright!",
-            "setup.py": "import CORE",
-            "README": "core readme",
-            "twisted": {"__init__.py": "twisted",
-                        "python": {"__init__.py": "python",
-                                   "roots.py": "roots!"},
-                        "plugin.py": "plugin",
-                        "plugins": {"twisted_whatever.py": "include!",
-                                    "cred.py": "include!"}},
-            "doc": {"howto": {"template.tpl": self.template,
-                              "index.html": howtoOutput,
-                              "tutorial": {"index.html": tutorialOutput}},
-                    "specifications": {"index.html": specOutput},
-                    "upgrades": {"index.html": upgradeOutput},
-                    "examples": {"foo.py": "foo.py"},
-                    "index.html": indexOutput},
-            "bin": {"twistd": "TWISTD"},
-            }
-
-        self.createStructure(self.rootDir, structure)
-        outputFile = self.builder.buildCore("8.0.0")
-        self.assertExtractedStructure(outputFile, outStructure)
-
-
-    def test_apiBaseURL(self):
-        """
-        DistributionBuilder builds documentation with the specified
-        API base URL.
-        """
-        apiBaseURL = "http://%s"
-        builder = DistributionBuilder(self.rootDir, self.outputDir,
-                                      apiBaseURL=apiBaseURL)
-        loreInput, loreOutput = self.getArbitraryLoreInputAndOutput(
-            "0.3.0", apiBaseURL=apiBaseURL)
-        structure = {
-            "LICENSE": "copyright!",
-            "twisted": {"web": {"__init__.py": "import WEB",
-                                "topfiles": {"setup.py": "import WEBINST"}}},
-            "doc": {"web": {"howto": {"index.xhtml": loreInput}},
-                    "core": {"howto": {"template.tpl": self.template}}
-                    }
-            }
-
-        outStructure = {
-            "LICENSE": "copyright!",
-            "setup.py": "import WEBINST",
-            "twisted": {"web": {"__init__.py": "import WEB"}},
-            "doc": {"howto": {"index.html": loreOutput}}}
-
-        self.createStructure(self.rootDir, structure)
-        outputFile = builder.buildSubProject("web", "0.3.0")
-        self.assertExtractedStructure(outputFile, outStructure)
-
-
-
 class BuildAllTarballsTest(DistributionBuilderTestBase):
     """
     Tests for L{DistributionBuilder.buildAllTarballs}.
@@ -2524 +1600 @@
         newsBuilder.buildAll = builds.append
         newsBuilder.main(["/foo/bar/baz"])
         self.assertEquals(builds, [FilePath("/foo/bar/baz")])
+
+
+

twisted/topfiles/setup.py

@@ -17 +17 @@
 from twisted import copyright
 from twisted.python.dist import setup, ConditionalExtension as Extension
 from twisted.python.dist import getPackages, getDataFiles, getScripts
-from twisted.python.dist import twisted_subprojects
+from twisted.python._dist import twisted_subprojects