lex.py

def lex(line):
    """
    Consumes a line and yields tokens for numbers and relevant
    symbols and words.
    """
    for s in re.split('[^a-zA-Z0-9_%$/.]+', line):
        # Skip empty strings. We don't need them.
        if not s:
            continue

        if re.match('^-?[0-9]+$', s):
            yield Token.Integer(s)
        elif re.match('^\/[0-9]+$', s):
            yield Token.Denominator(s)
        elif re.match('^\.$', s):
            yield Token.Symbol(s)
        else:
            yield s