[Twisted-Python] Uploading multiple files using ftpclient in Twisted

Jaepyoung Kim jaepyoung.kim at gmail.com
Sat Jul 10 01:40:11 EDT 2010


Dear twisted-python expert,

I am build engineer and I need to upload binary to ftp server.
Binary files are about 30 and each file size is about 30M.

The current script is uploading using ftplib and it takes time about 1 hour.
I want to change this script to use twisted asynchronous function.
I thought if I use asynchronous function in twisted like following,
then file uploading will be executed in parallel.
But this was executed sequentially. Uploading second file starts afer
completing first file upload.
Could you check what was wrong in my source code? Or Am I wrong in
understanding asynchronous function?


        for file in fileList:
    	ftpClient.cwd("/tmp").addCallbacks(uploadFiles, fail,
callbackArgs=(file,ftpClient))

Thanks,
Jaepyoung



Full source code:

def filesucess(ab):
    print ab

def uploadFiles(result, file,ftpClient):
    d1,d2=ftpClient.storeFile('/tmp/'+file)
    d1.addCallback(cbStore, file).addErrback(fileTransferFail)
    d1.addCallback(filesucess)
    return d2



def fileTransferFail(failure):
    failure.printTraceback()
    reactor.stop()


def showBuffer(result, bufferProtocol):
    print 'Got data:'
    print bufferProtocol.buffer.getvalue()


class Options(usage.Options):
    optParameters = [['host', 'h', 'localhost'],
                     ['port', 'p', 21],
                     ['username', 'u', 'user'],
                     ['password', None, 'password'],
                     ['passive', None, 0],
                     ['debug', 'd', 1],
                    ]
def cbStore(consumer, filename):
    fs = FileSender()
    print filename+" cbstror"
    d = fs.beginFileTransfer(open(filename, 'r'), consumer)
    d.addCallback(lambda _: consumer.finish()).addErrback(fileTransferFail)
    return d

def run():
    # Get config
    config = Options()
    config.parseOptions()
    config.opts['port'] = int(config.opts['port'])
    config.opts['passive'] = int(config.opts['passive'])
    config.opts['debug'] = int(config.opts['debug'])

    # Create the client
    FTPClient.debug = config.opts['debug']
    creator = ClientCreator(reactor, FTPClient, config.opts['username'],
                            config.opts['password'],
passive=config.opts['passive'])
    print config.opts['password']
    creator.connectTCP(config.opts['host'],
config.opts['port'],timeout=10).addCallback(connectionMade).addErrback(connectionFailed)
    reactor.run()


def connectionFailed(f):
    print "Connection Failed:", f
    reactor.stop()

def connectionMade(ftpClient):
    # Get the current working directory
    ftpClient.pwd().addCallbacks(success, fail)
    fileList = os.listdir('./temp')
    for file in fileList:
    	ftpClient.cwd("/tmp").addCallbacks(uploadFiles, fail,
callbackArgs=(file,ftpClient))
    print "connectionmade"



More information about the Twisted-Python mailing list