lexer.py

def analyze(code):
    tokenizer = _lexemes.scanner(code)
    tokenpairs = []
    while 1:
        token = tokenizer.match()
        if not token:
            break
        tokenpairs.append(tuple([token.group(token.lastindex), _tokendesc[token.lastindex - 1]]))
    return tokenpairs

if __name__ == "__main__":
    code = "SELECT col_1, COUNT(col_2) FROM tb_1 WHERE col_1 = col_2"
    tokens = analyze(code)
    for t in tokens:
        print t