Assign Lexer

gistfile1.py

# -*- coding: utf-8 -*-
import sys
import decimal

import ply.lex as lex

class AssignLexer(object):

    t_ignore = ' \t\r\f\v'
    t_ignore_comment = r'\#.*'
    def t_error(self, t):
        print("Illegal character '%s'" % t.value[0])
        t.lexer.skip(1)

    tokens = (
        'ASSIGN',
        'PLUS',
        'MINUS',
        'MULT',
        'DIV',
        'NAME',
        'NUMBER',
        'STRING',
        'WS',
        'NEWLINE',
    )

    t_ASSIGN = r'='
    t_PLUS = r'\+'
    t_MINUS = r'-'
    t_MULT = r'\*'
    t_DIV = r'/'

    def t_NAME(self, t):
        r'[a-zA-Z_][a-zA-Z0-9_]*'
        t.type = "NAME"
        return t

    def t_NUMBER(self, t):
        r"(\d+(\.\d*)?|\.\d+)([eE][-+]? \d+)?"
        # t.value = int(t.value)
        t.value = decimal.Decimal(t.value)
        return t

    def t_STRING(self, t):
        r"(?:'([^\\']+|\\'|\\\\)*'|\"([^\\\"]+|\\\"|\\\\)*\")"
        t.value = t.value[1:-1].decode("string-escape")
        return t

    def t_WS(self, t):
        r' [ ]+ '
        if t.lexer.at_line_start and t.lexer.paren_count == 0:
            return t

    def t_newline(self, t):
        r'\n+'
        t.lexer.lineno += len(t.value)

    def token(self):
        return self.lex.token()

    def get_lexer(self, debug=0):
        return lex.lex(object=self, debug=debug)

    def input(self, text, debug=0):
        self.lex = self.get_lexer(debug=debug)
        self.line_head_pos = 0
        self.lex.input(text)

    def test(self, text, debug=0):
        try:
            self.input(text, debug=debug)
            while True:
                tok = self.token()
                if not tok:
                    break
                print(tok)
        except SyntaxError:
            ex = sys.exc_info()[1]
            print(ex)

if __name__ == '__main__':
    # Test
    data = """
    3 + 2 * 4 / 2 # number plus test.
    """
    AssignLexer().test(data, debug=0)
    print("------------------------------------------")
    print("------------------------------------------")
    data = '''
    'name' # string test.
    '''
    AssignLexer().test(data, debug=0)