Incomplete, experimental bitbake recipe parsing with shlex

test.py

#!/usr/bin/env python

from shlex import shlex
import string
import os

class ParseError(Exception):
    def __init__(self, lex, message):
        self.lex = lex
        self.lineno = lex.lineno
        self.filename = lex.infile
        super(ParseError, self).__init__(self, lex.error_leader() + message)

class UnexpectedToken(ParseError):
    def __init__(self, lex, expected, got):
        self.expected = expected
        self.got = got
        super(UnexpectedToken, self).__init__(self, lex, 'Expected "%s", got "%s"' % (expected, got))

class UnexpectedEOF(ParseError):
    def __init__(self, lex):
        super(UnexpectedEOF, self).__init__(self, lex, "Unexpected End of File")
        
class IncludeError(ParseError):
    pass


def start(lex):
    global python

    key = lex.get_token()
    if key is None:
        return end(lex)
    elif key == "":
        # empty line
        return start(lex)

    python = False
    if key == "python":
        python = True
        key = lex.get_token()
        if key is None:
            raise UnexpectedEOF(lex)

    op = lex.get_token()
    if op == ".":
        return dot(key, lex)
    elif op == "+":
        return plus(key, lex)
    elif op == "=":
        return equals(key, lex, " ")
    elif op == "()":
        return parens(key, lex, python)
    elif op == "[":
        return openbracket(key, lex)
    elif op is None:
        raise UnexpectedEOF(lex)
    elif key == "include":
        lex.push_token(op)
        return include(lex)
    elif key == "require":
        lex.push_token(op)
        return include(lex, True)
    else:
        raise UnexpectedToken(lex, (".=", "+=", "=", "()"), op)

def openbracket(key, lex):
    flag = lex.get_token()
    if flag is None:
        raise UnexpectedEOF()

    endbracket = lex.get_token()
    if endbracket is None:
        raise UnexpectedEOF(lex)
    elif endbracket != "]":
        raise UnexpectedToken(lex, "]", endbracket)

    eq = lex.get_token()
    if eq != "=":
        raise UnexpectedToken(lex, "=", eq)

    return equals(key, lex, "[%s] " % flag)

def parens(key, lex, python):
    brace = lex.get_token()
    if brace == "{":
        return funcstart(key, lex, python)
    else:
        raise UnexpectedToken(lex, "{", brace)

def funcstart(key, lex, python):
    whitespace = lex.whitespace
    wordchars = lex.wordchars
    lex.wordchars = string.printable
    lex.whitespace = "\r\n"
    value = []
    for token in lex:
        if token == "}":
            print("%s%s () {\n%s\n}" % (python and "python " or "", key, "\n".join(value)))
            lex.wordchars = wordchars
            lex.whitespace = whitespace
            return start(lex)
        elif token is None:
            lex.wordchars = wordchars
            lex.whitespace = whitespace
            raise UnexpectedEOF(lex)
        else:
            value.append(token)
    return end(lex)

def dot(key, lex):
    eq = lex.get_token()
    if eq == "=":
        return concat(key, lex)
    raise UnexpectedToken(lex, "=", eq)

def plus(key, lex):
    eq = lex.get_token()
    if eq == "=":
        return addword(key, lex)
    raise UnexpectedToken(lex, "=", eq)

def equals(key, lex, mode):
    wordchars = lex.wordchars
    lex.wordchars = string.printable
    whitespace = lex.whitespace
    lex.whitespace = "\r\n"
    value = lex.get_token()
    lex.wordchars = wordchars
    lex.whitespace = whitespace
    if value is None:
        raise UnexpectedEOF(lex)
    else:
        value = value.strip()
    print("%s%s= \"%s\"" % (key, mode, value))
    return start(lex)

def concat(key, lex):
    return equals(key, lex, " .")

def addword(key, lex):
    return equals(key, lex, " +")


def include(lex, required = False):
    filename = lex.get_token()
    if filename is None:
        raise UnexpectedEOF(lex)
    elif filename == "":
        raise IncludeError(lex, "Unspecified filename for include")

    if os.path.exists(filename):
        parse(filename)
    elif required:
        raise IncludeError(lex, "Unable to include %s: file not found" % filename)

    return start(lex)

def end(lex):
    lex.instream.close()

def parse(filename):
    lex = shlex(open(filename, "r"), filename, True)
    lex.whitespace_split = False
    lex.wordchars = string.digits + string.letters + "~!@#$%*_\:;?,./-+()"
    lex.whitespace = " \t\r\n"
    lex.quotes = "\"'"
    #lex.debug = 1

    try:
        start(lex)
    except ParseError, e:
        import sys, linecache

        # How many lines of context from the file to display
        lines = 5

        sys.stderr.write("Parsing error: %s\n" % e)
        sys.stderr.write("%s context (%s lines):\n" % (filename, lines))
        for lineno in xrange(e.lineno - lines, e.lineno + lines):
            sys.stderr.write(linecache.getline(filename, lineno))

        raise SystemExit(1)

    raise SystemExit(0)

parse("testfile")

# TODO:
# - require
# - addtask
# - EXPORT_FUNCTIONS
# - inherit

testfile

A = "foo"
B .= 'bar'
D += meh moo

foo () {
whee
foo
        heh
}

python bar () {
    print("hi mom")
}

do_clean[nostamp] = "1"

require foo

include testinclude

baz="heh"

#baz
#C =
#bar () {

testinclude

weincluded = "yay!"