Crawler Sample

crawler.py

#!/usr/bin/python
# -*- coding: utf-8 -*-
import hashlib
import os
from bs4 import BeautifulSoup
import requests

__author__ = 'engine'

def get_photos(url):
    req = requests.get(url)
    dir = 'photos'
    if not os.path.exists(dir):
        os.makedirs(dir)
    if req.status_code == requests.codes.ok:
        soup = BeautifulSoup(req.text)
        div_img_list = soup.find_all('div', class_='photo_item inline-block')
        for div in div_img_list:
            img_url = div.a.img['src']
            print 'Download %s ' % img_url,
            name = os.path.join(dir, hashlib.md5(img_url).hexdigest() + os.path.splitext(img_url)[1])
            r = requests.get(img_url, stream=True)
            if r.status_code == requests.codes.ok:
                with open(name, 'wb') as f:
                    for chunk in r.iter_content():
                        f.write(chunk)
                print '[OK]'
            else:
                print '[Fail]'

if __name__ == '__main__':
    get_photos('http://photo.xuite.net/linopqr/19469116*1')