PKGBUILD parser

LICENSE

Copyright (c) 2010 PyroPeter

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

notbash-parser.py

#!/usr/bin/env python2

# This is a parser for bash-style variable assignments.
# It fully (?) supports the syntax w/o braces, the ${}-syntax
# is implemented very hackish. (There are also detailed
# TODO-comments in various places)

# This is meant for Archlinux' AUR to be used to parse
# PKGBUILD's properly.

# At the moment I have problems to find PKGBUILD's that are challenging
# for this script.

import re

reName = re.compile(r"([\w_][\w\d_]*)")
reAssignment = re.compile(r"([\w_][\w\d_]*)=")

def bashGlobToRegex(glob):
    # Reference: bash(1) "Pathname Expansion"
    # TODO:
    # * characater classes, equivalence classes, that other foo
    #   ( [:upper:], [=c=], [.symbol.]
    #   First should be easy to implement, python's regexes even have
    #   similar classes (\w, \W, \d, \D, \s, \S)
    #   Afaik, the first syntax is used rarely, the following two are
    #   used _never_. No urge to implement them. 
    # * extended globs:
    #   ( ?(pattern), *(pattern), +(pattern), etc...)
    #   Never seen someone use thiese, no urge to implement them.
    # * bash may have different handling of \:
    #   This code just ignores the following characters' special meaning
    #   and lets it match itself. Bash may evaluate e.g. "\a" as "\a"
    #   and not as "a". (But both should eval "\*" as "*".)
    #   UPDATE: This should now be solved. But it was done with magic.
    
    res = ""
    ptr = 0
    while ptr < len(glob):
        if glob[ptr] == "\\":
            ptr += 1
            res += re.escape(glob[ptr])
        elif glob[ptr] == "*":
            res += ".*?"
        elif glob[ptr] == "?":
            res += "."
        elif glob[ptr] == "[":
            ptr += 1
            res += "["
            if glob[ptr] == "^" or glob[ptr] == "!":
                res += "^"
                ptr += 1
            if glob[ptr] == "]":
                res += "]"
                ptr += 1
            if glob[ptr-1] in "[^!" and glob[ptr] == "\\":
                ptr += 1
            while glob[ptr] != "]":
                if glob[ptr] == "\\" and glob[ptr+1] in "wsdbaWSDBAZ":
                    res += "\\"
                res += glob[ptr]
                ptr += 1
            res += "]"
        else:
            res += re.escape(glob[ptr])
        ptr += 1
    return res

def expandParams(symbols, text):
    # Reference: bash(1) "Parameter Expansion"
    # Done:
    # * ${foo}
    # * ${foo/glob/substitute}
    #
    # Not done:
    # * all other syntaxes
    # * support for arrays will not be needed
    # * support for 2nd-level indirection will not be needed
    #   ( foo="bar"; bar="baz"; ${!foo} -> "baz" )
    
    res = ""
    ptr = 0
    while ptr < len(text):
        if text[ptr] == "\\":
            ptr += 1
        elif text[ptr] == "$":
            ptr += 1
            
            # are there braces?
            if text[ptr] == "{":
                ptr += 1
                name = reName.match(text, ptr)
                ptr = name.end()
                
                if text[ptr] == "/":
                    # Pattern substitution
                    # ${parameter/pattern/string}
                    
                    ptr += 1
                    
                    # check if all occurences should be substituted
                    suball = False
                    if text[ptr] == "/":
                        suball = True
                        ptr += 1
                    
                    # fetch the pattern
                    pattern = ""
                    while text[ptr] != "/" and text[ptr] != "}":
                        if text[ptr] == "\\" and text[ptr+1] in "\\}/":
                            ptr += 1
                        pattern += text[ptr]
                        ptr += 1
                    
                    # fetch the substitute
                    substitute = ""
                    if text[ptr] != "}": # there might be no substitute
                        ptr += 1
                        while text[ptr] != "}":
                            if text[ptr] == "\\":
                                ptr += 1
                            substitute += text[ptr]
                            ptr += 1
                    
                    if pattern.startswith("#"):
                        align = 1
                        pattern = pattern[1:]
                    elif pattern.startswith("\\#"):
                        pattern = pattern[1:]
                    elif pattern.endswith("%%"):
                        align = 2
                        pattern = pattern[:-1]
                    elif pattern.endswith("\\%%"):
                        pattern = pattern[:-1]
                    else:
                        align = 0
                    regex = ((align == 1 and "^" or "") +
                            bashGlobToRegex(pattern) +
                            (align == 2 and "$" or ""))
                    
                    res += re.sub(regex, substitute,
                        symbols.get(name.group(1), ""),
                        0 if suball else 1)
                else:
                    # 'normal' expansion
                    res += symbols.get(name.group(1), "")
            else:
                name = reName.match(text, ptr)
                res += symbols.get(name.group(1), "")
                ptr = name.end()
        else:
            res += text[ptr]
        ptr += 1
    return res

def parseStr(symbols, line, ptr):
    # TODO:
    # * This does not parse escapes
    res = ""
    if line[ptr] == "'":
        ptr += 1
        while line[ptr] != "'":
            res += line[ptr]
            ptr += 1
        ptr += 1
    elif line[ptr] == '"':
        ptr += 1
        while line[ptr] != '"':
            res += line[ptr]
            ptr += 1
        res = expandParams(symbols, res)
        ptr += 1
    else:
        while (len(line) > ptr) and (not line[ptr] in " \t)"):
            res += line[ptr]
            ptr += 1
        res = expandParams(symbols, res)
    return ptr, res

def parseFile(fileh):
    # TODO:
    # * Lines are not parsed like they should be.
    #   This is very unlikely to break the variable assignments
    #   at the begining of the file, but will break if a 'string'
    #   spanning multiple lines is encountered and it uses \ at the end
    #   of the line (e.g. a bash script in a bash script)
    lines = [""]

    for line in fileh:
        line = line[:-1]
        if not line: continue
        lines[-1] += line
        if line[-1] != "\\":
           lines.append("")
    
    symbols = {}
    i = 0
    while i < len(lines):
        line = lines[i]
        i += 1
        
        assignment = reAssignment.match(line)
        if not assignment:
            continue
        ptr = assignment.end()
        
        # the parser relys on proper syntax. syntax errors are
        # catched here
        try:
            if line[ptr] == "(":
                symbols[assignment.group(1)] = []
                ptr += 1
                while True:
                    while (ptr < len(line)) and (line[ptr] != ")"):
                        if not line[ptr] in " \t":
                            ptr, val = parseStr(symbols, line, ptr)
                            symbols[assignment.group(1)].append(val)
                        else:
                            ptr += 1
                    if (ptr < len(line)) and (line[ptr] == ")"): break
                    if len(lines) == i: break
                    line = lines[i]
                    i += 1
                    ptr = 0
            else:
                ptr, symbols[assignment.group(1)] = parseStr(
                    symbols, line, ptr)
        except IndexError:
            print >> sys.stderr, "Syntax error, continuing"
            continue
    return symbols

if __name__ == '__main__':
    import sys
    
    class cooldict(dict):
        def __init__(self, foo):
            self.foo = foo
        def __getitem__(self, key):
            res = self.foo.get(key, "(missing)")
            if hasattr(res, 'sort'):
                res = ', '.join(res)
            return res
    
    if len(sys.argv) < 2:
        print "Usage: %s <filename or '-'>"%(sys.argv[0])
        exit(1)
    
    res = cooldict(parseFile(
        open(sys.argv[1]) if sys.argv[1] != "-" else sys.stdin))
    print """%(pkgname)s %(pkgver)s
%(pkgdesc)s

Licenses: %(license)s
Architectures: %(arch)s

Dependencies: %(depends)s
    for make: %(makedepends)s

Source: %(source)s"""%res

Thanks! What your license for this?

@pombredanne I have added a license. (MIT License)

As I'm sure you've noticed, the quality of this code is extremely questionable.