武动乾坤小说采集脚本

wudongqiankun.py

# -*- coding:utf-8 -*-
import sys, database, feedparser

reload(sys)
sys.setdefaultencoding('utf-8')

db = database.Connection(host='127.0.0.1', database='wudongqiankun', user='root', password='password')

class Dict():
    def __init__(self, d):
        self.d = d
    def __getattr__(self, attr):
        return self.d.get(attr, None)

class Rssfeed(object):
    def __init__(self, feed_url):
        d = feedparser.parse(feed_url)
        self.entries = d.entries
        self.length = len(d.entries)

for i in xrange(1, 77):
    feed_url = 'http://www.321xz.com/feed/page/%s/'%i
    feed1 = Rssfeed(feed_url)
    _list = []
    for counter in range(0, feed1.length):
        item = feed1.entries[counter]
        _list.append((item.title, Dict(item.content[0]).value.replace('http://www.321xz.com/', 'http://wudongqiankun.au81.com/')))

    db.executemany("INSERT INTO `wudongqiankun_contents` (`title`, `text`, `created`, `modified`, `authorId`, `status`, `allowComment`, `allowPing`, `allowFeed`) VALUES (%s, %s, UNIX_TIMESTAMP(), UNIX_TIMESTAMP(), 1, 'draft', 0, 1, 0)", _list)