Similar to re.Scanner but works as a coroutine, yielding tokens as found

yieldscanner.py

import re

class NotAToken(Exception):
    pass

class YieldScanner(re.Scanner):
    def scan(self, inputstr):
        match = self.scanner.scanner(inputstr).match
        pos = 0
        while True:
            m = match()
            if m is None:
                if pos < len(inputstr):
                    raise NotAToken(inputstr[pos:])
                break
            endpos = m.end()
            if pos == endpos:
                raise EmptyToken(inputstr[pos:])
            yield (self.lexicon[m.lastindex - 1][1], m)
            pos = endpos