[Twisted-web] Deferreds and Loops

hancock.robert at gmail.com hancock.robert at gmail.com
Tue Oct 21 15:41:25 EDT 2008


I want to get the contents of a list of URLs, and if the getPage is  =

successful then I want to write it to a file. If it fails, I want to log a  =

message. In addition, I would like to have this process run every five  =

minutes.

The problem is how to specify a yes/no scenario when the callbacks take  =

arguments. Also, is the best way to loop to just put FeederProtocol.start()=
  =

in a loop? any ehlp would be greatly appreciated.

while true:
get page
if error:
log
else:
write to file

sleep 5 minutes

class FeederProtocol(object):

def get_page_error(self, _failure, log, feed_name):
""" Failure while getting page. """
msg =3D 'While getting %s: %s' % (feed_name, _failure.getTraceback())
log.error(msg)
_failure.trap(RuntimeError) # Caused chain to stop
raise(RuntimeError)

get_page(url):
return client.getPage(url, timeout=3D30)

def page_to_file(self, feed, name):
try:
fnout =3D os.path.join(app_conf['download_dir'], name)
fout =3D open(fnout, 'w')
except IOError, e:
log.error('Opening %s: %s' % (fnout, e))
return

try:
fout.write(feed)
except IOError, e:
log.error('Wrting to %s: %s' % (fnout, e))
finally:
fout.close()

def start(self, urls):
for url in urls:
d =3D defer.succeed(self.printStatus())
d.addCallback(self.get_page, feed, log, app_conf)
d.addCallbacks((self.page_to_file, log, app_conf), (self.get_page_error,  =

log, feed[0]))

class FeederFactory(protocol.ClientFactory):
protocol =3D FeederProtocol()

def __init__(self, url_list):
urls =3D url_list
self.start(self.urls)

def start(self, urls):
protocol.start(urls)


if __name__=3D=3D"__main__":
name =3D 'mytest'
urls =3D ['www.example.com', 'www.example2.com']

f =3D FeederFactory(urls)
reactor.run()
-------------- next part --------------
An HTML attachment was scrubbed...
URL: http://twistedmatrix.com/pipermail/twisted-web/attachments/20081021/ca=
4a4b95/attachment.htm


More information about the Twisted-web mailing list