locales_page_text parser

page_text_loc8.py

import sys
from urllib.request import urlopen
from time import sleep

"""
select concat('(',it.entry,', ', pt.entry, ', \'http://ru.wowhead.com/object=', it.entry,'\'),\\') as arr
from gameobject_template it
LEFT JOIN page_text pt ON it.data0 = pt.entry
left join locales_page_text lt ON pt.entry = lt.entry
where type=9 and ifnull(data0,0) <> 0 and lt.entry is null and pt.entry is not null

union all

select concat('(',it.entry,', ', pt.entry, ', \'http://ru.wowhead.com/item=', it.entry,'\'),\\') as arr
from item_template it
LEFT JOIN page_text pt ON it.pagetext = pt.entry
left join locales_page_text lt ON pt.entry = lt.entry
where ifnull(pagetext,0) <> 0 and lt.entry is null and pt.entry is not null;
"""

entryList = [\
    # insert query output here
(191649, 3218, 'http://ru.wowhead.com/object=191649'),\
(194155, 3544, 'http://ru.wowhead.com/object=194155'),\
(12564, 1771, 'http://ru.wowhead.com/item=12564'),\
(18401, 2637, 'http://ru.wowhead.com/item=18401'),\
(28046, 2981, 'http://ru.wowhead.com/item=28046'),\
(5562, 953, 'http://ru.wowhead.com/item=5562'),\
(5563, 954, 'http://ru.wowhead.com/item=5563'),\
(6280, 695, 'http://ru.wowhead.com/item=6280'),\
(28107, 2982, 'http://ru.wowhead.com/item=28107'),\
(1356, 44, 'http://ru.wowhead.com/item=1356'),\
(5353, 360, 'http://ru.wowhead.com/item=5353'),\
             ]

def clenuap(content):
    return  content.strip() \
        .replace("<br />", "\\n") \
        .replace("&lt;",   "<") \
        .replace("&gt;",   ">") \
        .replace("&nbsp;", " ") \
        .replace("&quot;", "\"") \
        .replace("&amp;",  "&") \
        .replace("'",      "\'") \
        .replace("\"",     "\\\"")

with open("d:\page_loc8.sql", "w", -1, "utf-8") as sql_file:
    def parse_page(entry, page, content, start_pattern, end_pattern):
        startpos = content.find(start_pattern)
        endpos = content.find(end_pattern, startpos)
        if (startpos > -1 and endpos > -1):
            startpos += len(start_pattern)
            pages = content[startpos:endpos].split("','")
            for p in pages:               
                sql = "INSERT IGNORE INTO `locales_page_text` (entry, text_loc8) VALUES (%d, '%s');\n" % (page, clenuap(p))
                sql_file.writelines(sql)
                sql_file.flush()
                print(sql)
                page+=1
        else:
            print("No matches found for page: %d -> '%s'" % (page, start_pattern))
        pass

    for rec in entryList:
        entry = rec[0]
        page  = rec[1]
        url   = rec[2]
        try:
            print("Open: %s" % url)
            content = urlopen(url)\
                .read()\
                .decode("utf-8")

            parse_page(entry, page, content, "', pages: ['", "']})")
        except Exception as ex:
            print("Err: %d (%s)" % (entry, ex))
        sleep(1)

print("Done!")