Async Batching with Twisted: A Walkthrough

01-deferred-list.py

from twisted.internet import defer, reactor
from twisted.web.client import getPage

def listCallback(results):
  print results

def finish(ign):
  reactor.stop()

def test():
  d1 = getPage('http://www.google.com')
  d2 = getPage('http://yahoo.com')
  dl = defer.DeferredList([d1, d2])
  dl.addCallback(listCallback)
  dl.addCallback(finish)

test()
reactor.run()

02-deferred-list-results.py

from twisted.internet import defer, reactor
from twisted.web.client import getPage

def listCallback(results):
  for isSuccess, content in results:
    print "Successful? %s" % isSuccess
    print "Content Length: %s" % len(content)

def finish(ign):
  reactor.stop()

def test():
  d1 = getPage('http://www.google.com')
  d2 = getPage('http://yahoo.com')
  dl = defer.DeferredList([d1, d2])
  dl.addCallback(listCallback)
  dl.addCallback(finish)

test()
reactor.run()

03-page-callbacks.py

from twisted.internet import defer, reactor
from twisted.web.client import getPage

def pageCallback(result):
  return len(result)

def listCallback(result):
  print result

def finish(ign):
  reactor.stop()

def test():
  d1 = getPage('http://www.google.com')
  d1.addCallback(pageCallback)
  d2 = getPage('http://yahoo.com')
  d2.addCallback(pageCallback)
  dl = defer.DeferredList([d1, d2])
  dl.addCallback(listCallback)
  dl.addCallback(finish)

test()
reactor.run()

04-results-with-structure.py

from twisted.internet import defer, reactor
from twisted.web.client import getPage

def pageCallback(result):
  data = {
    'length': len(result),
    'content': result[:10],
    }
  return data

def listCallback(result):
  for isSuccess, data in result:
    if isSuccess:
      print "Call to server succeeded with data %s" % str(data)

def finish(ign):
  reactor.stop()

def test():
  d1 = getPage('http://www.google.com')
  d1.addCallback(pageCallback)
  d2 = getPage('http://yahoo.com')
  d2.addCallback(pageCallback)
  dl = defer.DeferredList([d1, d2])
  dl.addCallback(listCallback)
  dl.addCallback(finish)

test()
reactor.run()

05-passing-values.py

from twisted.internet import defer, reactor
from twisted.web.client import getPage

def pageCallback(result, url):
  data = {
    'length': len(result),
    'content': result[:10],
    'url': url,
    }
  return data

def getPageData(url):
  d = getPage(url)
  d.addCallback(pageCallback, url)
  return d

def listCallback(result):
  for isSuccess, data in result:
    if isSuccess:
      print "Call to %s succeeded with data %s" % (data['url'], str(data))

def finish(ign):
  reactor.stop()

def test():
  d1 = getPageData('http://www.google.com')
  d2 = getPageData('http://yahoo.com')
  dl = defer.DeferredList([d1, d2])
  dl.addCallback(listCallback)
  dl.addCallback(finish)

test()
reactor.run()

06-error-checking.py

from twisted.internet import defer, reactor
from twisted.web.client import getPage

urls = [
  'http://yahoo.com',
  'http://www.google.com',
  'http://www.google.com/MicrosoftRules.html',
  'http://bogusdomain.com',
  ]

def pageCallback(result, url):
  data = {
    'length': len(result),
    'content': result[:10],
    'url': url,
    }
  return data

def pageErrback(error, url):
  return {
    'msg': error.getErrorMessage(),
    'err': error,
    'url': url,
    }

def getPageData(url):
  d = getPage(url, timeout=5)
  d.addCallback(pageCallback, url)
  d.addErrback(pageErrback, url)
  return d

def listCallback(result):
  for ignore, data in result:
    if data.has_key('err'):
      print "Call to %s failed with data %s" % (data['url'], str(data))
    else:
      print "Call to %s succeeded with data %s" % (data['url'], str(data))

def finish(ign):
  reactor.stop()

def test():
  deferreds = []
  for url in urls:
    d = getPageData(url)
    deferreds.append(d)
  dl = defer.DeferredList(deferreds, consumeErrors=1)
  dl.addCallback(listCallback)
  dl.addCallback(finish)

test()
reactor.run()

07-deferred-semaphore.py

from twisted.internet import defer, reactor
from twisted.web.client import getPage

maxRun = 1

urls = [
  'http://twistedmatrix.com',
  'http://yahoo.com',
  'http://www.google.com',
  ]

def listCallback(results):
  for isSuccess, result in results:
    print len(result)

def finish(ign):
  reactor.stop()

def test():
  deferreds = []
  sem = defer.DeferredSemaphore(maxRun)
  for url in urls:
    d = sem.run(getPage, url)
    deferreds.append(d)
  dl = defer.DeferredList(deferreds)
  dl.addCallback(listCallback)
  dl.addCallback(finish)

test()
reactor.run()

08-cooperator.py

from twisted.internet import defer, reactor, task
from twisted.web.client import getPage

maxRun = 2

urls = [
  'http://twistedmatrix.com',
  'http://yahoo.com',
  'http://www.google.com',
  ]

def pageCallback(result):
  print len(result)
  return result

def doWork():
  for url in urls:
    d = getPage(url)
    d.addCallback(pageCallback)
    yield d

def finish(ign):
  reactor.stop()

def test():
  deferreds = []
  coop = task.Cooperator()
  work = doWork()
  for i in xrange(maxRun):
    d = coop.coiterate(work)
    deferreds.append(d)
  dl = defer.DeferredList(deferreds)
  dl.addCallback(finish)

test()
reactor.run()