lightweight regex lexer

lex.py

import re

tokenizer = re.compile(r"""
    (?P<string>     " (?: [^\\"] | \\. )* "     ) |
    (?P<number>     0[Xx][0-9A-Fa-f]+ | [0-9]+  ) |
    (?P<comment>    \# .*? (?= \n | $ )         ) |
    (?P<ident>      [A-Za-z_] [A-Za-z0-9_]*     ) |
    (?P<paren>      [()]                        ) |
    (?P<brace>      [{}]                        ) |
    (?P<bracket>    [][]                        ) |
    (?P<eol>        ;                           ) |
    (?P<operator>   [-+*/!=<>^&|.,~?:]+         ) |
    (?P<whitespace> [ \t\n]+                    ) |
    (?P<invalid>    .                           )
""", re.VERBOSE | re.MULTILINE)

def lex(s):
    for m in tokenizer.finditer(s):
        k, v = [(k, v) for k, v in m.groupdict().items() if v is not None][0]
        if k == 'invalid':
            raise SyntaxError("invalid data at position %d" % m.start())
        elif k not in ('whitespace', 'comment'):
            yield (k, v)
    yield ('eof', None) # makes writing parsers a little bit easier