netdesign

g++ -c -pipe -O2 -Wall -W -D_REENTRANT  -DQT_NO_DEBUG -DQT_CORE_LIB -DQT_SHARED -I../../../lib/qt4/mkspecs/linux-g++ -I. -I../../../lib/qt4/include/QtCore -I../../../lib/qt4/include/QtCore -I../../../lib/qt4/include -I. -I. -o qtdirs.o qtdirs.cpp
../../../lib/qt4/include/QtCore/qatomic_i386.h: In function ‘int q_atomic_test_and_set_ptr(volatile void*, void*, void*)’:
../../../lib/qt4/include/QtCore/qatomic_i386.h:58: error: cast from ‘void*’ to ‘int’ loses precision
../../../lib/qt4/include/QtCore/qatomic_i386.h:59: error: cast from ‘void*’ to ‘int’ loses precision
../../../lib/qt4/include/QtCore/qatomic_i386.h: In function ‘void* q_atomic_set_ptr(volatile void*, void*)’:
../../../lib/qt4/include/QtCore/qatomic_i386.h:98: error: cast from ‘void*’ to ‘int’ loses precision
make: *** [qtdirs.o] Error 1
Error: Failed to determine the layout of your Qt installation. Try again using

netdesign

Traceback (most recent call last):
  File "parser.py", line 6, in <module>
    print('%s' % elem[1].text.encode('utf-8'))
AttributeError: 'NoneType' object has no attribute 'encode'

netdesign

from xml.dom.pulldom import START_ELEMENT, parse
from lxml import etree

src = 'content.rdf.u8'
doc = parse(src)

for event, node in doc:
        if event == START_ELEMENT and node.localName == "ExternalPage":
                doc.expandNode(node)
                parser = etree.XMLParser(dtd_validation=False,recover=True,no_network=True)

netdesign

import lxml.etree as tr
context = tr.iterparse('content.rdf.u8', events=('end',), tag='ExternalPage')

for event, elem in context:
        print('%s\n' % elem.text.encode('utf-8'))
        elem.clear()

netdesign

import lxml.etree as tr
context = tr.iterparse('content.rdf.u8', events=('end',), tag='{http://dmoz.org/rdf/}ExternalPage')

for event, elem in context:
        print('%s\n' % elem.text.encode('utf-8'))
        elem.clear()
        while elem.getprevious() is not None:
                del elem.getparent()[0]

netdesign

oursql.CollatedWarningsError: (None, 'query caused warnings', [(<class 'oursql.Warning'>, (u"Incorrect string value: '\\xE2\\x99\\xA5' for column 'title' at row 1", 1366L))])

netdesign

#!/usr/bin/env python

import lxml.etree as tr
import oursql
import time
import signal
import sys
from multiprocessing import Process, Queue
from collections import deque

netdesign

Process Process-2:
Traceback (most recent call last):
  File "/usr/lib/python2.6/multiprocessing/process.py", line 232, in _bootstrap
    self.run()
  File "/usr/lib/python2.6/multiprocessing/process.py", line 88, in run
    self._target(*self._args, **self._kwargs)
TypeError: 'Process' object is not callable
^CError in atexit._run_exitfuncs:
Traceback (most recent call last):
  File "/usr/lib/python2.6/atexit.py", line 24, in _run_exitfuncs

netdesign

MainParser parses an XML document like this:

<ExternalPage about="http://www.awn.com/"><d:Title></d:Title><d:Description></d:Description><priority>1</priority><topic>Top/Arts/Animation</topic></ExternalPage>
<ExternalPage about="http://www.awn.com/"><d:Title></d:Title><d:Description></d:Description><priority>1</priority><topic>Top/Arts/Animation</topic></ExternalPage>
[......many other nodes]

then queues a single <ExternalPage><tags/data></ExternalPage>

then xmlParser (but also xmlSecondParser) get attribs and texts from the first node of the Queue and then INSERTS them in DB

netdesign

+-------+--------------+-----------------+------+-----+---------+-------+---------------------------------+---------+
| Field | Type         | Collation       | Null | Key | Default | Extra | Privileges                      | Comment |
+-------+--------------+-----------------+------+-----+---------+-------+---------------------------------+---------+
| url   | varchar(800) | utf8_general_ci | YES  |     | NULL    |       | select,insert,update,references |         |
| title | varchar(800) | utf8_general_ci | YES  |     | NULL    |       | select,insert,update,references |         |
| descr | varchar(800) | utf8_general_ci | YES  |     | NULL    |       | select,insert,update,references |         |
+-------+--------------+-----------------+------+-----+---------+-------+---------------------------------+---------+