Created
September 25, 2011 16:10
-
-
Save netdesign/1240770 to your computer and use it in GitHub Desktop.
Simple multiprocessing XML parser
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env python | |
| import lxml.etree as tr | |
| import oursql | |
| import time | |
| import signal | |
| import sys | |
| from multiprocessing import Process, Queue | |
| from collections import deque | |
| errors = 0 | |
| def mainParser(queue): | |
| context = tr.iterparse('content.rdf.u8', events=('end',), tag='{http://dmoz.org/rdf/}ExternalPage') | |
| XMLNS_D = '{http://purl.org/dc/elements/1.0/}' | |
| for event, elem in context: | |
| if(queue.qsize() > 12000): | |
| time.sleep(5) | |
| queue.put(tr.tostring(elem)) | |
| #print "Main "+str(queue.qsize()) | |
| elem.clear() | |
| while elem.getprevious() is not None: | |
| del elem.getparent()[0] | |
| def xmlParser(queue): | |
| XMLNS_D = '{http://purl.org/dc/elements/1.0/}' | |
| conn = oursql.connect("HIDDEN") | |
| curs = conn.cursor(try_plain_query=False) | |
| curs.execute("SET NAMES utf8") | |
| while(queue.qsize() >= 0): | |
| elem = tr.fromstring(queue.get()) | |
| title = elem.find(XMLNS_D + 'Title').text | |
| if title is not None: | |
| title = title | |
| descr = elem.find(XMLNS_D + 'Description').text | |
| if descr is not None: | |
| descr = descr | |
| url = elem.attrib['about'] | |
| try: | |
| curs.execute('INSERT INTO table (url, title, descr) VALUES(?, ?, ?)', (url, title, descr)) | |
| except: | |
| print "Error" | |
| def xmlSecondParser(queue): | |
| XMLNS_D = '{http://purl.org/dc/elements/1.0/}' | |
| conn = oursql.connect("HIDDEN") | |
| curs = conn.cursor(try_plain_query=False) | |
| curs.execute("SET NAMES utf8") | |
| while(queue.qsize() >= 0): | |
| elem = tr.fromstring(queue.get()) | |
| title = elem.find(XMLNS_D + 'Title').text | |
| if title is not None: | |
| title = title | |
| descr = elem.find(XMLNS_D + 'Description').text | |
| if descr is not None: | |
| descr = descr | |
| url = elem.attrib['about'] | |
| try: | |
| curs.execute('INSERT INTO table (url, title, descr) VALUES(?, ?, ?)', (url, title, descr)) | |
| except: | |
| print "Error" | |
| if __name__ == '__main__': | |
| q = Queue() | |
| xmlParser = Process(target=xmlParser, args=(q,)) #XML PARSER PARSES DATA FROM NODES | |
| xmlSecondParser = Process(target=xmlSecondParser, args=(q,)) #XML PARSER PARSES DATA FROM NODES | |
| mainParser = Process(target=mainParser, args=(q,)) #MAIN XML PARSE SINGLE NODES | |
| mainParser.start() | |
| xmlParser.start() | |
| xmlSecondParser.start() | |
| print "main "+str(mainParser.pid) | |
| print "one "+str(xmlParser.pid) | |
| print "sec "+str(xmlSecondParser.pid) | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment