Python XML Parser with single thread

lashou_parser_single.py

# coding=utf-8
import sys
import xml.etree.cElementTree as cElementTree
from pyquery import PyQuery
import time

reload(sys)
sys.setdefaultencoding('utf-8')

start_time = time.time()

q = []

PATH = (
    ('offer title', 'title', None, None),
    ('offer image_url', 'img_url', None, None),
    ('offer url', 'url', None, None),
    ('offer publish_cities city', 'city', "List",
     lambda x: '全国' if len(x) > 1 else x[0]),
    ('offer start_timestamp', 'begintime', None, None),
    ('offer end_timestamp', 'endtime', None, None),
    ('offer original_price', 'value', None, None),
    ('offer current_price', 'price', None, None),
    ('offer sales_number', 'curnumber', None, None),
    ('offer shops shop_info shop_name', "shop_name", "List",
     lambda x: ','.join(x)),
    ('offer shops shop_info shop_address', "shop_address", "List",
     lambda x: ','.join(x)),
    ('offer shops shop_info shop_phone', "shop_phone", "List",
     lambda x: ','.join(x)),
    ('offer shops shop_info longitude', "long", "List", lambda x: ','.join(x)),
    ('offer shops shop_info latitude', "lat", "List", lambda x: ','.join(x)),

)


def boss():
    infile = 'test/lashou_global.xml'
    context = cElementTree.iterparse(infile, events=('end',))
    context = iter(context)
    event, root = context.next()

    for event, elem in context:
        if elem.tag == "offer":
            node = cElementTree.tostring(elem)
            q.append(node)
            root.clear()
    print time.time() - start_time


def parse(xml):
    d = PyQuery(xml, parser='xml')
    res = dict()
    for item in PATH:
        path = item[0]
        alias = item[1]
        item_type = item[2]
        dealer = item[3]
        if item_type == 'List':
            res[alias] = dealer([node.text for node in d(path)])
        elif item_type is None:
            res[alias] = d(path).text()
        else:
            res[alias] = None
            print 'No type settings...'
    return res


def worker(name):
    print '%s is working...' % name
    deal = 0
    for task in q:
        parse(task)
        deal += 1
        if deal % 100 == 0:
            print '%s has dealed %d deals...' % (name, deal)
    print time.time() - start_time

if __name__ == '__main__':
    boss()
    worker('only one')
    print 'Done'