foo.py

from twisted.internet.defer import Deferred


class Page(object):
    def __init__(self, results, morefunc):
        self.results = results
        self._morefunc = morefunc

    def next_page(self):
        return self._morefunc()


class PageMaker(object):
    def __init__(self, all_results, page_size):
        self._all_results = all_results[:]
        self._page_size = page_size

    def get_page(self):
        d = Deferred()
        if not self._all_results:
            d.callback(None)
        else:
            results = self._all_results[:self._page_size]
            self._all_results[:self._page_size] = []
            d.callback(Page(results, self.get_page))
        return d

print "=============== 1 ==============="
pm1 = PageMaker(range(10), 3)

# In the real world, this deferred probably won't have a result yet. We can
# cheat here because get_page() always calls d.callback().
page_d = pm1.get_page()
print "Deferred page:", page_d
page = page_d.result

while page is not None:
    print "Page:", page, page.results
    page_d = page.next_page()
    print "Deferred page:", page_d
    page = page_d.result


# This doesn't really explore the true depths of the problems because our
# deferreds always fire synchronously. You can either think through the
# behaviour when next_page() returns a deferred that will only fire later or
# you can write a Page implementation that injects some async delay and run it
# with a reactor.

class PageIter(object):
    def __init__(self, page, horribleness):
        self._page = page
        self._deferreds = []
        assert horribleness in ["infinite", "exception"]
        self._horribleness = horribleness

    def __iter__(self):
        return self

    def next(self):
        """
        Create a deferred that will fire with the next page in the sequence,
        call the function that populates the next deferred with a witing page
        if we have one, and return the deferred we just made.
        """
        d = Deferred()
        self._deferreds.append(d)
        self._populate_d()
        return d

    def _next_page_cb(self, page):
        """
        We have a new page, so stash it and call the function that populates
        the next deferred with the new page.
        """
        self._page = page
        self._populate_d()

    def _populate_d(self):
        """
        If we have a non-empty queue of deferreds and a waiting page, pop the
        first deferred out of the queue and put the waiting page in it.
        Otherwise do nothing.
        """
        if not self._deferreds:
            # No deferreds to put the result into.
            return
        if self._page is None:
            if self._horribleness == "exception":
                # If we don't have a new page waiting, raise an exception.
                raise RuntimeError("no waiting page")
            # No result to put in the next deferred.
            return
        # Pull the first deferred off the front of the queue and put the
        # current page into it.
        d = self._deferreds.pop(0)
        self._page, page = None, self._page
        d.callback(page)
        page.next_page().addCallback(self._next_page_cb)


def walk_pages(page, horribleness):
    try:
        for page_d in PageIter(page, horribleness):
            print "Deferred page:", page_d
            assert page_d.called, "page_d will never get a result"
            page = page_d.result
            print "Page:", page, page.results
    except Exception as e:
        print "ERROR!", e


print "=============== 2 ==============="
page = PageMaker(range(10), 3).get_page().result
walk_pages(page, "exception")


print "=============== 3 ==============="
page = PageMaker(range(10), 3).get_page().result
walk_pages(page, "infinite")