http chunked encoding iterator

http_stream.py

class HTTPStream(object):
    """Given a URL which returns chunk-encoded results,
    constructs an iterator over those results (assuming
    individual records are newline-separated)."""
    def __init__(self, url, username="", password=""):
        self.prev_linepart = ""
        self.http_client = HTTPClient()
        self.http_client.fetch(
            url=url,
            auth_username=username,
            auth_password=password,
            streaming_callback=self._handle_data)

    def __iter__(self):
        return self

    def next(self):
        """Returns next line from buffer populated by _handle_data()."""
        return self.line_iter.next()

    def _iter_lines(self):
        while True:
            line = (yield)
            yield line

    def _handle_data(self, data):
        lines = (self.prev_linepart + data).split("\r\n")
        self.prev_linepart = lines[-1]
        lines = lines[:-1]
        if not lines:
            if self.line_iter:
                self.line_iter.close()
            return
        if not self.line_iter:
            self.line_iter = self._iter_lines()
            self.line_iter.next() # prime for first send()
        # yield to iterator function
        for line in lines:
            self.line_iter.send(line)