Ticket #5960: lockfile-r1.patch

File lockfile-r1.patch, 19.5 KB (added by jamesbroadhead, 3 years ago)
  • setup3.py

    diff --git setup3.py setup3.py
    index cc54546..d76bce1 100644
    modules = [ 
    7474    "twisted.python.deprecate",
    7575    "twisted.python.failure",
    7676    "twisted.python.filepath",
     77    "twisted.python.lockfile",
    7778    "twisted.python.log",
    7879    "twisted.python.monkey",
    7980    "twisted.python.randbytes",
  • twisted/internet/defer.py

    diff --git twisted/internet/defer.py twisted/internet/defer.py
    index d16b51d..2b8865a 100644
    from sys import exc_info 
    2525from functools import wraps
    2626
    2727# Twisted imports
    28 from twisted.python.compat import _PY3, comparable, cmp
    29 from twisted.python import log, failure
     28from twisted.python.compat import cmp, comparable
     29from twisted.python import lockfile, log, failure
    3030from twisted.python.deprecate import warnAboutFunction
    3131
    3232
    class DeferredQueue(object): 
    14851485
    14861486
    14871487
    1488 # Re-add to Python 3 in #5960:
    1489 if not _PY3:
    1490     from twisted.python import lockfile
    1491 
    1492     class AlreadyTryingToLockError(Exception):
    1493         """
    1494         Raised when L{DeferredFilesystemLock.deferUntilLocked} is called twice on a
    1495         single L{DeferredFilesystemLock}.
    1496         """
    14971488
     1489class AlreadyTryingToLockError(Exception):
     1490    """
     1491    Raised when L{DeferredFilesystemLock.deferUntilLocked} is called twice on a
     1492    single L{DeferredFilesystemLock}.
     1493    """
    14981494
    14991495
    1500     class DeferredFilesystemLock(lockfile.FilesystemLock):
    1501         """
    1502         A L{FilesystemLock} that allows for a L{Deferred} to be fired when the lock is
    1503         acquired.
     1496class DeferredFilesystemLock(lockfile.FilesystemLock):
     1497    """
     1498    A L{FilesystemLock} that allows for a L{Deferred} to be fired when the lock is
     1499    acquired.
    15041500
    1505         @ivar _scheduler: The object in charge of scheduling retries. In this
    1506             implementation this is parameterized for testing.
     1501    @ivar _scheduler: The object in charge of scheduling retries. In this
     1502        implementation this is parameterized for testing.
    15071503
    1508         @ivar _interval: The retry interval for an L{IReactorTime} based scheduler.
     1504    @ivar _interval: The retry interval for an L{IReactorTime} based scheduler.
    15091505
    1510         @ivar _tryLockCall: A L{DelayedCall} based on C{_interval} that will manage
    1511             the next retry for aquiring the lock.
     1506    @ivar _tryLockCall: A L{DelayedCall} based on C{_interval} that will manage
     1507        the next retry for aquiring the lock.
    15121508
    1513         @ivar _timeoutCall: A L{DelayedCall} based on C{deferUntilLocked}'s timeout
    1514             argument.  This is in charge of timing out our attempt to acquire the
    1515             lock.
    1516         """
    1517         _interval = 1
    1518         _tryLockCall = None
    1519         _timeoutCall = None
     1509    @ivar _timeoutCall: A L{DelayedCall} based on C{deferUntilLocked}'s timeout
     1510        argument.  This is in charge of timing out our attempt to acquire the
     1511        lock.
     1512    """
     1513    _interval = 1
     1514    _tryLockCall = None
     1515    _timeoutCall = None
    15201516
    15211517
    1522         def __init__(self, name, scheduler=None):
    1523             """
    1524             @param name: The name of the lock to acquire
    1525             @param scheduler: An object which provides L{IReactorTime}
    1526             """
    1527             lockfile.FilesystemLock.__init__(self, name)
     1518    def __init__(self, name, scheduler=None):
     1519        """
     1520        @param name: The name of the lock to acquire
     1521        @param scheduler: An object which provides L{IReactorTime}
     1522        """
     1523        lockfile.FilesystemLock.__init__(self, name)
    15281524
    1529             if scheduler is None:
    1530                 from twisted.internet import reactor
    1531                 scheduler = reactor
     1525        if scheduler is None:
     1526            from twisted.internet import reactor
     1527            scheduler = reactor
    15321528
    1533             self._scheduler = scheduler
     1529        self._scheduler = scheduler
    15341530
    15351531
    1536         def deferUntilLocked(self, timeout=None):
    1537             """
    1538             Wait until we acquire this lock.  This method is not safe for
    1539             concurrent use.
     1532    def deferUntilLocked(self, timeout=None):
     1533        """
     1534        Wait until we acquire this lock.  This method is not safe for
     1535        concurrent use.
    15401536
    1541             @type timeout: C{float} or C{int}
    1542             @param timeout: the number of seconds after which to time out if the
    1543                 lock has not been acquired.
     1537        @type timeout: C{float} or C{int}
     1538        @param timeout: the number of seconds after which to time out if the
     1539            lock has not been acquired.
    15441540
    1545             @return: a L{Deferred} which will callback when the lock is acquired, or
    1546                 errback with a L{TimeoutError} after timing out or an
    1547                 L{AlreadyTryingToLockError} if the L{deferUntilLocked} has already
    1548                 been called and not successfully locked the file.
    1549             """
    1550             if self._tryLockCall is not None:
    1551                 return fail(
    1552                     AlreadyTryingToLockError(
    1553                         "deferUntilLocked isn't safe for concurrent use."))
     1541        @return: a L{Deferred} which will callback when the lock is acquired, or
     1542            errback with a L{TimeoutError} after timing out or an
     1543            L{AlreadyTryingToLockError} if the L{deferUntilLocked} has already
     1544            been called and not successfully locked the file.
     1545        """
     1546        if self._tryLockCall is not None:
     1547            return fail(
     1548                AlreadyTryingToLockError(
     1549                    "deferUntilLocked isn't safe for concurrent use."))
    15541550
    1555             d = Deferred()
     1551        d = Deferred()
    15561552
    1557             def _cancelLock():
    1558                 self._tryLockCall.cancel()
    1559                 self._tryLockCall = None
    1560                 self._timeoutCall = None
     1553        def _cancelLock():
     1554            self._tryLockCall.cancel()
     1555            self._tryLockCall = None
     1556            self._timeoutCall = None
    15611557
    1562                 if self.lock():
    1563                     d.callback(None)
    1564                 else:
    1565                     d.errback(failure.Failure(
    1566                             TimeoutError("Timed out aquiring lock: %s after %fs" % (
    1567                                     self.name,
    1568                                     timeout))))
     1558            if self.lock():
     1559                d.callback(None)
     1560            else:
     1561                d.errback(failure.Failure(
     1562                        TimeoutError("Timed out aquiring lock: %s after %fs" % (
     1563                                self.name,
     1564                                timeout))))
    15691565
    1570             def _tryLock():
    1571                 if self.lock():
    1572                     if self._timeoutCall is not None:
    1573                         self._timeoutCall.cancel()
    1574                         self._timeoutCall = None
     1566        def _tryLock():
     1567            if self.lock():
     1568                if self._timeoutCall is not None:
     1569                    self._timeoutCall.cancel()
     1570                    self._timeoutCall = None
    15751571
    1576                     self._tryLockCall = None
     1572                self._tryLockCall = None
    15771573
    1578                     d.callback(None)
    1579                 else:
    1580                     if timeout is not None and self._timeoutCall is None:
    1581                         self._timeoutCall = self._scheduler.callLater(
    1582                             timeout, _cancelLock)
     1574                d.callback(None)
     1575            else:
     1576                if timeout is not None and self._timeoutCall is None:
     1577                    self._timeoutCall = self._scheduler.callLater(
     1578                        timeout, _cancelLock)
    15831579
    1584                     self._tryLockCall = self._scheduler.callLater(
    1585                         self._interval, _tryLock)
     1580                self._tryLockCall = self._scheduler.callLater(
     1581                    self._interval, _tryLock)
    15861582
    1587             _tryLock()
     1583        _tryLock()
    15881584
    1589             return d
     1585        return d
    15901586
    15911587
    15921588
  • twisted/python/lockfile.py

    diff --git twisted/python/lockfile.py twisted/python/lockfile.py
    index a044957..cb441d9 100644
    from time import time as _uniquefloat 
    1616from twisted.python.runtime import platform
    1717
    1818def unique():
    19     return str(long(_uniquefloat() * 1000))
     19    return str(int(_uniquefloat() * 1000))
    2020
    2121from os import rename
    2222if not platform.isWindows():
    else: 
    4040        def kill(pid, signal):
    4141            try:
    4242                OpenProcess(0, 0, pid)
    43             except pywintypes.error, e:
     43            except pywintypes.error as e:
    4444                if e.args[0] == ERROR_ACCESS_DENIED:
    4545                    return
    4646                elif e.args[0] == ERROR_INVALID_PARAMETER:
    else: 
    7070    def readlink(filename):
    7171        try:
    7272            fObj = _open(os.path.join(filename,'symlink'), 'rb')
    73         except IOError, e:
     73        except IOError as e:
    7474            if e.errno == errno.ENOENT or e.errno == errno.EIO:
    7575                raise OSError(e.errno, None)
    7676            raise
    class FilesystemLock: 
    125125        while True:
    126126            try:
    127127                symlink(str(os.getpid()), self.name)
    128             except OSError, e:
     128            except OSError as e:
    129129                if _windows and e.errno in (errno.EACCES, errno.EIO):
    130130                    # The lock is in the middle of being deleted because we're
    131131                    # on Windows where lock removal isn't atomic.  Give up, we
    class FilesystemLock: 
    134134                if e.errno == errno.EEXIST:
    135135                    try:
    136136                        pid = readlink(self.name)
    137                     except OSError, e:
     137                    except OSError as e:
    138138                        if e.errno == errno.ENOENT:
    139139                            # The lock has vanished, try to claim it in the
    140140                            # next iteration through the loop.
    141141                            continue
    142142                        raise
    143                     except IOError, e:
     143                    except IOError as e:
    144144                        if _windows and e.errno == errno.EACCES:
    145145                            # The lock is in the middle of being
    146146                            # deleted because we're on Windows where
    class FilesystemLock: 
    152152                    try:
    153153                        if kill is not None:
    154154                            kill(int(pid), 0)
    155                     except OSError, e:
     155                    except OSError as e:
    156156                        if e.errno == errno.ESRCH:
    157157                            # The owner has vanished, try to claim it in the next
    158158                            # iteration through the loop.
    159159                            try:
    160160                                rmlink(self.name)
    161                             except OSError, e:
     161                            except OSError as e:
    162162                                if e.errno == errno.ENOENT:
    163163                                    # Another process cleaned up the lock.
    164164                                    # Race them to acquire it in the next
  • twisted/test/test_defer.py

    diff --git twisted/test/test_defer.py twisted/test/test_defer.py
    index da787db..1438d78 100644
    import warnings 
    1111import gc, traceback
    1212import re
    1313
     14from twisted.python import failure, log
    1415from twisted.python.compat import _PY3
     16from twisted.internet import defer, reactor
     17from twisted.internet.task import Clock
    1518from twisted.trial import unittest
    16 from twisted.internet import defer
    17 from twisted.python import failure, log
     19
    1820
    1921
    2022class GenericError(Exception):
    class DeferredTestCase(unittest.SynchronousTestCase, ImmediateFailureMixin): 
    913915        d.addCallback(circularCallback)
    914916        d.callback("foo")
    915917
    916         warnings = self.flushWarnings([circularCallback])
    917         self.assertEqual(len(warnings), 1)
    918         warning = warnings[0]
     918        warnings_ = self.flushWarnings([circularCallback])
     919        self.assertEqual(len(warnings_), 1)
     920        warning = warnings_[0]
    919921        self.assertEqual(warning['category'], DeprecationWarning)
    920922        pattern = "Callback returned the Deferred it was attached to"
    921923        self.assertTrue(
    class OtherPrimitives(unittest.SynchronousTestCase, ImmediateFailureMixin): 
    20122014        self.assertEqual(len(done), 1)
    20132015
    20142016
    2015 # Enable on Python 3 as part of #5960:
    2016 if not _PY3:
    2017     from twisted.internet import reactor
    2018     from twisted.internet.task import Clock
     2017class DeferredFilesystemLockTestCase(unittest.TestCase):
     2018    """
     2019    Test the behavior of L{DeferredFilesystemLock}
     2020    """
     2021
     2022    def setUp(self):
     2023        self.clock = Clock()
     2024        self.lock = defer.DeferredFilesystemLock(self.mktemp(),
     2025                                                 scheduler=self.clock)
     2026
    20192027
    2020     class DeferredFilesystemLockTestCase(unittest.TestCase):
     2028    def test_waitUntilLockedWithNoLock(self):
    20212029        """
    2022         Test the behavior of L{DeferredFilesystemLock}
     2030        Test that the lock can be acquired when no lock is held
    20232031        """
     2032        d = self.lock.deferUntilLocked(timeout=1)
    20242033
    2025         def setUp(self):
    2026             self.clock = Clock()
    2027             self.lock = defer.DeferredFilesystemLock(self.mktemp(),
    2028                                                      scheduler=self.clock)
    2029 
     2034        return d
    20302035
    2031         def test_waitUntilLockedWithNoLock(self):
    2032             """
    2033             Test that the lock can be acquired when no lock is held
    2034             """
    2035             d = self.lock.deferUntilLocked(timeout=1)
    20362036
    2037             return d
     2037    def test_waitUntilLockedWithTimeoutLocked(self):
     2038        """
     2039        Test that the lock can not be acquired when the lock is held
     2040        for longer than the timeout.
     2041        """
     2042        self.failUnless(self.lock.lock())
    20382043
     2044        d = self.lock.deferUntilLocked(timeout=5.5)
     2045        self.assertFailure(d, defer.TimeoutError)
    20392046
    2040         def test_waitUntilLockedWithTimeoutLocked(self):
    2041             """
    2042             Test that the lock can not be acquired when the lock is held
    2043             for longer than the timeout.
    2044             """
    2045             self.failUnless(self.lock.lock())
     2047        self.clock.pump([1] * 10)
    20462048
    2047             d = self.lock.deferUntilLocked(timeout=5.5)
    2048             self.assertFailure(d, defer.TimeoutError)
     2049        return d
    20492050
    2050             self.clock.pump([1] * 10)
    20512051
    2052             return d
     2052    def test_waitUntilLockedWithTimeoutUnlocked(self):
     2053        """
     2054        Test that a lock can be acquired while a lock is held
     2055        but the lock is unlocked before our timeout.
     2056        """
     2057        def onTimeout(f):
     2058            f.trap(defer.TimeoutError)
     2059            self.fail("Should not have timed out")
    20532060
     2061        self.failUnless(self.lock.lock())
    20542062
    2055         def test_waitUntilLockedWithTimeoutUnlocked(self):
    2056             """
    2057             Test that a lock can be acquired while a lock is held
    2058             but the lock is unlocked before our timeout.
    2059             """
    2060             def onTimeout(f):
    2061                 f.trap(defer.TimeoutError)
    2062                 self.fail("Should not have timed out")
     2063        self.clock.callLater(1, self.lock.unlock)
     2064        d = self.lock.deferUntilLocked(timeout=10)
     2065        d.addErrback(onTimeout)
    20632066
    2064             self.failUnless(self.lock.lock())
     2067        self.clock.pump([1] * 10)
    20652068
    2066             self.clock.callLater(1, self.lock.unlock)
    2067             d = self.lock.deferUntilLocked(timeout=10)
    2068             d.addErrback(onTimeout)
     2069        return d
    20692070
    2070             self.clock.pump([1] * 10)
    20712071
    2072             return d
     2072    def test_defaultScheduler(self):
     2073        """
     2074        Test that the default scheduler is set up properly.
     2075        """
     2076        lock = defer.DeferredFilesystemLock(self.mktemp())
    20732077
     2078        self.assertEqual(lock._scheduler, reactor)
    20742079
    2075         def test_defaultScheduler(self):
    2076             """
    2077             Test that the default scheduler is set up properly.
    2078             """
    2079             lock = defer.DeferredFilesystemLock(self.mktemp())
    20802080
    2081             self.assertEqual(lock._scheduler, reactor)
     2081    def test_concurrentUsage(self):
     2082        """
     2083        Test that an appropriate exception is raised when attempting
     2084        to use deferUntilLocked concurrently.
     2085        """
     2086        self.lock.lock()
     2087        self.clock.callLater(1, self.lock.unlock)
    20822088
     2089        d = self.lock.deferUntilLocked()
     2090        d2 = self.lock.deferUntilLocked()
    20832091
    2084         def test_concurrentUsage(self):
    2085             """
    2086             Test that an appropriate exception is raised when attempting
    2087             to use deferUntilLocked concurrently.
    2088             """
    2089             self.lock.lock()
    2090             self.clock.callLater(1, self.lock.unlock)
     2092        self.assertFailure(d2, defer.AlreadyTryingToLockError)
    20912093
    2092             d = self.lock.deferUntilLocked()
    2093             d2 = self.lock.deferUntilLocked()
     2094        self.clock.advance(1)
    20942095
    2095             self.assertFailure(d2, defer.AlreadyTryingToLockError)
     2096        return d
    20962097
    2097             self.clock.advance(1)
    20982098
     2099    def test_multipleUsages(self):
     2100        """
     2101        Test that a DeferredFilesystemLock can be used multiple times
     2102        """
     2103        def lockAquired(ign):
     2104            self.lock.unlock()
     2105            d = self.lock.deferUntilLocked()
    20992106            return d
    21002107
     2108        self.lock.lock()
     2109        self.clock.callLater(1, self.lock.unlock)
    21012110
    2102         def test_multipleUsages(self):
    2103             """
    2104             Test that a DeferredFilesystemLock can be used multiple times
    2105             """
    2106             def lockAquired(ign):
    2107                 self.lock.unlock()
    2108                 d = self.lock.deferUntilLocked()
    2109                 return d
     2111        d = self.lock.deferUntilLocked()
     2112        d.addCallback(lockAquired)
    21102113
    2111             self.lock.lock()
    2112             self.clock.callLater(1, self.lock.unlock)
     2114        self.clock.advance(1)
    21132115
    2114             d = self.lock.deferUntilLocked()
    2115             d.addCallback(lockAquired)
    2116 
    2117             self.clock.advance(1)
    2118 
    2119             return d
     2116        return d
  • new file twisted/topfiles/5960.feature

    diff --git twisted/topfiles/5960.feature twisted/topfiles/5960.feature
    new file mode 100644
    index 0000000..9b5d7f3
    - +  
     1Port twisted.python.lockfile to Python 3, enabling twisted.python.defer.DeferredFilesystemLock and tests.
  • twisted/trial/util.py

    diff --git twisted/trial/util.py twisted/trial/util.py
    index a3103f0..3c8ed80 100644
    from __future__ import division, absolute_import, print_function 
    2323import sys
    2424from random import randrange
    2525
    26 from twisted.python.compat import _PY3
    2726from twisted.internet import defer, utils, interfaces
    2827from twisted.python.failure import Failure
    2928from twisted.python import deprecate, versions
    3029from twisted.python.filepath import FilePath
     30from twisted.python.lockfile import FilesystemLock
    3131
    3232__all__ = [
    3333    'DEFAULT_TIMEOUT_DURATION',
    class _Janitor(object): 
    157157        reactor = self._getReactor()
    158158        if interfaces.IReactorThreads.providedBy(reactor):
    159159            if reactor.threadpool is not None:
    160                 # Stop the threadpool now so that a new one is created. 
     160                # Stop the threadpool now so that a new one is created.
    161161                # This improves test isolation somewhat (although this is a
    162162                # post class cleanup hook, so it's only isolating classes
    163163                # from each other, not methods from each other).
    def _unusedTestDirectory(base): 
    379379        same name until the lock is released, either explicitly or by this
    380380        process exiting.
    381381    """
    382     from twisted.python.lockfile import FilesystemLock
    383382    counter = 0
    384383    while True:
    385384        if counter:
    def _unusedTestDirectory(base): 
    405404                counter += 1
    406405            else:
    407406                raise _WorkingDirectoryBusy()
    408 
    409 # Remove this, and move lockfile import, after ticket #5960 is resolved:
    410 if _PY3:
    411     del _unusedTestDirectory