好图爬虫

haotu.py

from urllib2 import urlopen, URLError
from urllib import urlretrieve
import threading
import re
import os

downroot = r'D:\luguan'
p1 = re.compile(r'src="img.+jpg')
p2 = re.compile(r'\d{1,}\.jpg')
prefix = 'http://www.lu-guan.com/xxoo/'

def clean_dir(top):
    for root, dirs, files in os.walk(top, topdown=False):
        for name in files:
            os.remove(os.path.join(root, name))
        for name in dirs:
            os.rmdir(os.path.join(root, name))

def down_pics():
    print('start downloading')

    todown = range(1, 31)
    logfile = os.path.join(downroot, 'log')

    if os.path.exists(logfile):
        log = open(logfile, 'r+')
        downed = log.read().split(' ')
    else:
        log = open(logfile, 'w')
        downed = []

    for i in range(1, 101):
        if str(i) in downed:
            continue;
        url = prefix + ('%03d' % i) + '.html'
        print 'downloading %s' % url
        try:
            if down_one_url(url, i) == 404:
                break;
        except:
            print 'error occured during downloading %s' % url
            continue
        downed.append(str(i))
        print '%d completed' % i
    
    content = ' '.join(downed)
    log.write(content)
    log.close()
    print('finished!')

def down_one_url(url, index):
    folder = downroot + '\\' + ('%03d' % index)
    if not os.path.exists(folder):
        os.makedirs(folder)
    else:
        clean_dir(folder)

    try:
        data = urlopen(url).read();
    except URLError, e:
        if e.code == 404:
            return 404

    piclist = p1.findall(data)
    task_threads=[]
    for item in piclist:
        pic_path = prefix + item[5:]
        filepath = downroot + '\\%03d' % index + '\\' + p2.findall(item)[0]
        t = threading.Thread(target = urlretrieve, args = (pic_path, filepath))
        task_threads.append(t)
        
    for task in task_threads:
        task.start()

    for task in task_threads:
        task.join(300)
    return 0

if __name__ == '__main__':
    if not os.path.exists(downroot):
        os.mkdir(downroot)
    down_pics();