parser_h.py

import os, re, sys;
import unicodedata
from BeautifulSoup import BeautifulSoup

#reload(sys)
#sys.setdefaultencoding('utf8')
#reload(sys)
#print(sys.getdefaultencoding())

class HtParser():
    def __init__(self, file, opcode):
        self.file = file;
        self.opcode = opcode;
        page = open(file, "r");
        self.soup = BeautifulSoup(page);
        self.title = self.getTitle();
        self.instrList = []
        self.encList = {}
    pass
    def ReadFile(self):
        tableList = self.soup.findAll("table");
        i=0
        for table in tableList:
            if i < 5:
                type = self.GetTableType(table);
                if type == "INSTR":
                    self.parseInstr(table);
                elif type == "INSTR2_":
                    self.parseInstr2(table);
                elif type == "ENC":
                    self.parseEnc(table);
            i+=1;
        return True

    def getTitle(self):
        tt = self.soup.find("title");
        try:
            return tt.text[tt.text.find(u'\u2014')+1:].strip()
        except:
            return str(tt);


    def GetTableType(self, table):
        for row in table.contents:
            if str.strip(str(row)) == "":
                continue;
            try:
                if row.contents[1].text.upper() == "OPCODE"\
                and row.contents[3].text.upper() == "INSTRUCTION"\
                and row.contents[5].text.upper() == "OP/EN"\
                and row.contents[7].text.upper() == "64-BIT MODE"\
                and row.contents[9].text.upper() == "COMPAT/LEG MODE"\
                and row.contents[11].text.upper() == "DESCRIPTION":
                    return "INSTR"
                elif row.contents[1].text.upper() == "OP/EN"\
                and row.contents[3].text.upper() == "OPERAND 1"\
                and row.contents[5].text.upper() == "OPERAND 2"\
                and row.contents[7].text.upper() == "OPERAND 3"\
                and row.contents[9].text.upper() == "OPERAND 4":
                    return "ENC"
                elif unicode(str(row.contents[1].text), "utf-8").startswith("Opcode/I")\
                and unicode(str(row.contents[3].text), "utf-8").startswith("Op/En")\
                and unicode(str(row.contents[5].text), "utf-8").startswith("64/32")\
                and unicode(str(row.contents[7].text), "utf-8").startswith("CPUID")\
                and unicode(str(row.contents[9].text), "utf-8").startswith("Description"):
                    return "INSTR2"
            except:
                return None

    def parseCell(self, inst):
        ret = ""
        for cont in inst.contents:
            if str(cont).startswith("<"):
                ret += " " + str(cont.text);
            else:
                ret += " " + str(cont);
        #ret = unicodedata.normalize('NFC',ret);
        return ret.strip().replace("  ", " ").replace("&amp;", "&")

    def parseInstr(self, table):
        for row in table.contents:
            if str.strip(str(row)) == "":
                continue;
            if row.contents[1].text.upper() == "OPCODE":
                continue;
            opcode = self.parseCell(row.contents[1]).replace('/ ', '/').replace(' +', '+');
            instr  = self.parseCell(row.contents[3]).replace(' , ', ', ').replace(' ,', ', ').replace(' *', '*').replace('*','').replace('i mm', 'imm');#.text;# parse
            mnem   = instr.split(' ')[0];
            instr  = instr[len(mnem):].strip();
            enc    = row.contents[5].text.upper();
            bit64  = row.contents[7].text;
            compat = row.contents[9].text;
            desc   = unicode(self.parseCell( row.contents[11]), "utf-8");
            cpuid  = "ANY"
            arx = ""
            self.instrList.append({"mnem":mnem, "opcode":opcode, "instr":instr, "enc":enc, "bit64":bit64, "cpuid":cpuid, "arx":arx, "compat":compat, "desc":desc});

    def parseInstr2(self, table):
        for row in table.contents:
            if str.strip(str(row)) == "":
                continue;
            if row.contents[1].text.startswith("Opcode/I"):
                continue;

            if len(row.contents[1].contents) == 2:
                opcode = self.parseCell(row.contents[1].contents[0]).replace('/ r', '/r').replace(' +', '+');
                instr = self.parseCell(row.contents[1].contents[1]).replace(' , ', ', ').replace(' ,', ', ').replace(' *', '*').replace('*','').replace('i mm', 'imm');#.text;# parse
                str(1)
            elif len(row.contents[1].contents) < 3:
                st = self.parseCell(row.contents[1].contents[1]);
                pos = st.find(self.opcode);
                if pos > -1:
                    opcode = st[:pos];
                    instr = st[pos:]
                else:
                    raise BaseException("**** ERROR "+ self.opcode);
                print("")
            else:
                opcode = self.parseCell(row.contents[1].contents[1]).replace('/ r', '/r').replace(' +', '+');
                instr = self.parseCell(row.contents[1].contents[3]).replace(' , ', ', ').replace(' ,', ', ').replace(' *', '*').replace('*','').replace('i mm', 'imm');#.text;# parse
            mnem   = instr.split(' ')[0];
            instr  = instr[len(mnem):].strip();
            enc   = row.contents[3].text.upper();
            arx   = row.contents[5].text;
            cpuid = row.contents[7].text;
            desc  = unicode(self.parseCell( row.contents[9]), "utf-8");
            self.instrList.append({"mnem":mnem, "opcode":opcode, "instr":instr, "enc":enc, "bit64":"", "cpuid":cpuid, "arx":arx, "compat":"", "desc":desc});

    def parseEnc(self, table):
        for row in table.contents:
            if str.strip(str(row)) == "":
                continue;
            if row.contents[1].text.upper() == "OP/EN":
                continue;
            type = row.contents[1].text.upper();
            op1 = row.contents[3].text;
            op2 = row.contents[5].text;
            op3 = row.contents[7].text;
            op4 = row.contents[9].text;
            if op1 != "NA":
                self.encList[type] = [ op1, op2, op3, op4 ]

    def getstr(self, row):
        try:
            return "%s" % row[6]
        except:
            return ""

    def getRows(self):
        l = []
        for row in self.instrList:
            ennc1 = "" if not self.encList.has_key(row["enc"]) else self.encList[row["enc"]][0];
            ennc2 = "" if not self.encList.has_key(row["enc"]) else self.encList[row["enc"]][1];
            ennc3 = "" if not self.encList.has_key(row["enc"]) else self.encList[row["enc"]][2];
            ennc4 = "" if not self.encList.has_key(row["enc"]) else self.encList[row["enc"]][3];
            ops = row["instr"].split(',');
            op1 = ops[0] if len(ops) > 0 else "";
            op2 = ops[1] if len(ops) > 1 else "";
            op3 = ops[2] if len(ops) > 2 else "";
            op4 = ops[3] if len(ops) > 3 else "";
            """
            l.append(u'{ "mnem":"%s", "opcode":"%s", "instr":"%s", "enc"="%s", "e1":"%s", "ex1":"%s", "e2":"%s", "ex2":"%s", "e3":"%s", "ex3":"%s", "e4":"%s", "ex4":"%s", "bit64":"%s", "cpuid":"%s", "arx":"%s", "compat":"%s", "title":"%s", "desc":"%s" },' % (
                row["mnem"],
                row["opcode"],
                row["instr"],
                row["enc"],
                op1,ennc1,
                op2,ennc2,
                op3,ennc3,
                op4,ennc4,
                row["bit64"],
                row["cpuid"],
                row["arx"],
                row["compat"],
                self.title,
                row["desc"]
                ))
            """
            l.append(u'%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s' % (
                row["mnem"],
                row["opcode"],
                row["instr"],
                row["enc"],
                op1,ennc1,
                op2,ennc2,
                op3,ennc3,
                op4,ennc4,
                row["bit64"],
                row["cpuid"],
                row["arx"],
                row["compat"],
                self.title,
                row["desc"]
                ))
        return l
pass

dir = "C:\\Users\\Koc\\Desktop\\html"
files = os.listdir(dir)
for fname in files:
    #try:
        opcpde = fname.split(".")[0]
        parser = HtParser(os.path.join(dir, fname), opcpde)
        #print("-----"+fname+"------")
        parser.ReadFile();
        #parser.getRows()
        if len(parser.instrList)>0:
            for l in parser.getRows():
                print(l)
        #else:
        #    print("#"+fname)
    #except BaseException as ex:
        #print("**ERROR: "+fname + "   " +str(ex))