generator_based_parsing.py

""" Generator-based versions of tokenize and read_from_tokens from Norving's "How to write a Lisp Interpreter in Python" """
from itertools import takewhile


def tokenize(chars):
    ''' Convert a string expression into a generator of tokens. '''
    return (t for t in chars.replace('(', ' ( ').replace(')', ' ) ').split())


def read_from_tokens_gen(tokens, current_token=None):
    if current_token:
        token = current_token
    else:
        token = next(tokens)
    if token == '(':
        L = []
        for t in takewhile(lambda x: x != ')', tokens):
            L.append(read_from_tokens_gen(tokens, current_token=t))
        return L
    elif token == ')':
        raise SyntaxError('unexpected )')
    else:   
        return token