[Twisted-Python] Re: Re: Re: Resume FTP file retrieval
Andrew Bennetts
andrew-twisted at puzzling.org
Wed Jan 7 06:30:06 MST 2004
On Tue, Jan 06, 2004 at 03:37:21PM -0000, Richard.Townsend at edl.uk.eds.com wrote:
> Hi Andrew,
>
> I have tried to follow the code in FTPClient.
You're brave ;)
> It looks like the _openDataConnection() method is responsible for sending
> the PORT/PASV before the RETR. However, I'm not sure how it could be
> safely modified to send a REST between the two requests.
You've found the right spot -- but as you've probably noticed, it's probably
the hairiest code in the entire class.
> I need this facility to be able to resume a partial download when
> transferring large files over an unreliable comms link.
>
> Would you or another of Twisted experts be able to help make this change?
I've attached a completely untested patch! Let me know how it goes.
-Andrew.
-------------- next part --------------
Index: twisted/protocols/ftp.py
===================================================================
RCS file: /cvs/Twisted/twisted/protocols/ftp.py,v
retrieving revision 1.100
diff -u -r1.100 ftp.py
--- twisted/protocols/ftp.py 13 Dec 2003 01:12:09 -0000 1.100
+++ twisted/protocols/ftp.py 7 Jan 2004 13:28:09 -0000
@@ -1706,13 +1706,13 @@
else:
return None
- def receiveFromConnection(self, command, protocol):
+ def receiveFromConnection(self, commands, protocol):
"""
Retrieves a file or listing generated by the given command,
feeding it to the given protocol.
- @param command: string of an FTP command to execute then receive the
- results of (e.g. LIST, RETR)
+ @param command: list of strings of FTP commands to execute then receive
+ the results of (e.g. LIST, RETR)
@param protocol: A L{Protocol} *instance* e.g. an
L{FTPFileListProtocol}, or something that can be adapted to one.
Typically this will be an L{IConsumer} implemenation.
@@ -1721,9 +1721,9 @@
"""
protocol = IProtocol(protocol)
wrapper = ProtocolWrapper(protocol, Deferred())
- return self._openDataConnection(command, wrapper)
+ return self._openDataConnection(commands, wrapper)
- def sendToConnection(self, command):
+ def sendToConnection(self, commands):
"""XXX
@returns: A tuple of two L{Deferred}s:
@@ -1733,14 +1733,15 @@
- L{Deferred} list of control-connection responses.
"""
s = SenderProtocol()
- r = self._openDataConnection(command, s)
+ r = self._openDataConnection(commands, s)
return (s.connectedDeferred, r)
- def _openDataConnection(self, command, protocol):
+ def _openDataConnection(self, commands, protocol):
"""
This method returns a DeferredList.
"""
- cmd = FTPCommand(command, public=1)
+ cmds = [FTPCommand(command, public=1) for command in commands]
+ cmdsDeferred = DeferredList([cmd.deferred for cmd in cmds])
if self.passive:
# Hack: use a mutable object to sneak a variable out of the
@@ -1766,7 +1767,7 @@
self.queueCommand(pasvCmd)
pasvCmd.deferred.addCallback(doPassive).addErrback(self.fail)
- results = [cmd.deferred, pasvCmd.deferred, protocol.deferred]
+ results = [cmdsDeferred, pasvCmd.deferred, protocol.deferred]
d = DeferredList(results, fireOnOneErrback=1)
# Ensure the connection is always closed
@@ -1799,12 +1800,13 @@
portCmd.fail = lambda error: error
# Ensure that the connection always gets closed
- cmd.deferred.addErrback(lambda e, pc=portCmd: pc.fail(e) or e)
+ cmdsDeferred.addErrback(lambda e, pc=portCmd: pc.fail(e) or e)
- results = [cmd.deferred, portCmd.deferred, portCmd.transferDeferred]
+ results = [cmdsDeferred, portCmd.deferred, portCmd.transferDeferred]
d = DeferredList(results, fireOnOneErrback=1)
- self.queueCommand(cmd)
+ for cmd in cmds:
+ self.queueCommand(cmd)
return d
def generatePortCommand(self, portCmd):
@@ -1845,7 +1847,7 @@
# Escape newline characters
return string.replace(path, '\n', '\0')
- def retrieveFile(self, path, protocol):
+ def retrieveFile(self, path, protocol, offset=0):
"""Retrieve a file from the given path
This method issues the 'RETR' FTP command.
@@ -1855,14 +1857,18 @@
@param path: path to file that you wish to receive.
@param protocol: a L{Protocol} instance.
+ @param offset: offset to start downloading from
@returns: L{Deferred}
"""
- return self.receiveFromConnection('RETR ' + self.escapePath(path), protocol)
+ cmds = ['RETR ' + self.escapePath(path)]
+ if offset:
+ cmds.insert(0, ('REST ' + str(offset))
+ return self.receiveFromConnection(, protocol)
retr = retrieveFile
- def storeFile(self, path):
+ def storeFile(self, path, offset=0):
"""Store a file at the given path.
This method issues the 'STOR' FTP command.
@@ -1873,8 +1879,10 @@
is completely transferred.
- L{Deferred} list of control-connection responses.
"""
-
- return self.sendToConnection('STOR ' + self.escapePath(path))
+ cmds = ['STOR ' + self.escapePath(path)]
+ if offset:
+ cmds.insert(0, ('REST ' + str(offset))
+ return self.sendToConnection(cmds)
stor = storeFile
More information about the Twisted-Python
mailing list