Skip to content

Instantly share code, notes, and snippets.

@netdesign
Created September 25, 2011 16:10
Show Gist options
  • Select an option

  • Save netdesign/1240770 to your computer and use it in GitHub Desktop.

Select an option

Save netdesign/1240770 to your computer and use it in GitHub Desktop.
Simple multiprocessing XML parser
#!/usr/bin/env python
import lxml.etree as tr
import oursql
import time
import signal
import sys
from multiprocessing import Process, Queue
from collections import deque
errors = 0
def mainParser(queue):
context = tr.iterparse('content.rdf.u8', events=('end',), tag='{http://dmoz.org/rdf/}ExternalPage')
XMLNS_D = '{http://purl.org/dc/elements/1.0/}'
for event, elem in context:
if(queue.qsize() > 12000):
time.sleep(5)
queue.put(tr.tostring(elem))
#print "Main "+str(queue.qsize())
elem.clear()
while elem.getprevious() is not None:
del elem.getparent()[0]
def xmlParser(queue):
XMLNS_D = '{http://purl.org/dc/elements/1.0/}'
conn = oursql.connect("HIDDEN")
curs = conn.cursor(try_plain_query=False)
curs.execute("SET NAMES utf8")
while(queue.qsize() >= 0):
elem = tr.fromstring(queue.get())
title = elem.find(XMLNS_D + 'Title').text
if title is not None:
title = title
descr = elem.find(XMLNS_D + 'Description').text
if descr is not None:
descr = descr
url = elem.attrib['about']
try:
curs.execute('INSERT INTO table (url, title, descr) VALUES(?, ?, ?)', (url, title, descr))
except:
print "Error"
def xmlSecondParser(queue):
XMLNS_D = '{http://purl.org/dc/elements/1.0/}'
conn = oursql.connect("HIDDEN")
curs = conn.cursor(try_plain_query=False)
curs.execute("SET NAMES utf8")
while(queue.qsize() >= 0):
elem = tr.fromstring(queue.get())
title = elem.find(XMLNS_D + 'Title').text
if title is not None:
title = title
descr = elem.find(XMLNS_D + 'Description').text
if descr is not None:
descr = descr
url = elem.attrib['about']
try:
curs.execute('INSERT INTO table (url, title, descr) VALUES(?, ?, ?)', (url, title, descr))
except:
print "Error"
if __name__ == '__main__':
q = Queue()
xmlParser = Process(target=xmlParser, args=(q,)) #XML PARSER PARSES DATA FROM NODES
xmlSecondParser = Process(target=xmlSecondParser, args=(q,)) #XML PARSER PARSES DATA FROM NODES
mainParser = Process(target=mainParser, args=(q,)) #MAIN XML PARSE SINGLE NODES
mainParser.start()
xmlParser.start()
xmlSecondParser.start()
print "main "+str(mainParser.pid)
print "one "+str(xmlParser.pid)
print "sec "+str(xmlSecondParser.pid)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment