爬虫：百度搜索 关键字，返回指定页面范围内所有条目的题目和网站链接

getsites.py

#! /usr/bin/env python
# -*- coding: utf-8 -*-
# @Last Modified time: 2016-06-27 17:03:01
import requests
from lxml import html as HTML
import os
import codecs


class KeyWordSite(object):
    url_format = 'http://www.baidu.com/s?wd=%s&pn=%d'
    title_css = "h3 > a"
    links_css = "div.f13 > a.c-showurl"
    result_count = "div > div.nums"
    filter_sites = ["baidu.com", "weibo.com", "qq.com", "iqiyi.com", "sohu.com", "letv.com", "163.com",
                    "pptv.com", "toutiao.com"]

    headers = {"Host": "www.baidu.com",
               "Connection": "keep - alive",
               "Pragma": "no - cache",
               "Cache - Control": "no - cache",
               "Upgrade - Insecure - Requests": "1",
               "User-Agent": ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5)"
                              "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.84 Safari/537.36"),

               "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
               "Accept-Encoding": "gzip, deflate, sdch",
               "Accept-Language": "zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4"
               }

    def __init__(self, key_word, start_page, end_page):
        self.key_word = key_word
        self.start_page = start_page
        self.end_page = end_page
        self.count = ""
        # results keep (links, titles) tuple.
        self.results = []
        self.get_results()

    def __str__(self):
        self.extract_count()
        return self.count

    def _make_baidu_url_(self, page):
        """Create the search url of baidu

        @:param page: The search result page we want to get.
        Page > 25 may be useless, so just ignore it.
        """
        if page > 25 or page < 0:
            page = 0
        if page == 0:
            page = 1
        return self.url_format % (self.key_word, (page - 1) * 10)

    def _get_baidu_page_(self, page):
        """Crawl the specified page.
        """
        req_url = self._make_baidu_url_(page)
        res_page = requests.get(req_url, headers=self.headers)
        if res_page.status_code != 200:
            print "Get search result from baidu failed!  Something must be wrong."
            exit(-1)
        page_content = res_page.text
        page_tree = HTML.document_fromstring(page_content)
        return page_tree

    def extract_count(self):
        page_tree = self._get_baidu_page_(0)
        self.count = page_tree.cssselect(self.result_count)[
            0].text_content().encode("utf-8")

    def _extract_results_(self, page_tree):
        """Extract the sites from the html source content"""
        title_tags = page_tree.cssselect(self.title_css)
        links_tags = page_tree.cssselect(self.links_css)
        result_links = [result.text_content().encode("utf-8")
                        for result in links_tags]
        result_title = [result.text_content().encode("utf-8")
                        for result in title_tags]
        return result_links, result_title

    def _valid_result_(self, link):
        # Filter the sites have to many child domain.  Just use at mostly
        # 2-level domain.
        if len(link.split(".")) > 3:
            return False
        if any(map(lambda x: x in link, self.filter_sites)):
            return False

        return True

    @staticmethod
    def normalize_links(results):
        link_l = results[0].split("/")
        return link_l[0], results[1]

    def get_results(self):
        for page in range(self.start_page, self.end_page + 1):
            page_tree = self._get_baidu_page_(page)
            results = self._extract_results_(page_tree)
            self.results.extend(zip(results[0], results[1]))

        # Filter those sites such as baike.baidu.com
        self.results = list(
            filter(lambda x: self._valid_result_(x[0]), self.results))
        self.results = list(
            map(lambda x: KeyWordSite.normalize_links(x), self.results))

    @staticmethod
    def get_processed_links(results):
        """Convert links like wangluo.sdcit.cn to www.sdcit.cn
        """
        links = list(map(lambda x: x[0], results))
        titles = list(map(lambda x: x[1], results))
        new_links = []
        for l in links:
            l_split = l.split(".")
            if len(l_split) == 3 and l_split[0] != "www":
                l_split[0] = "www"
            new_links.append(".".join(l_split))
        return zip(new_links, titles)


class BatchSites(object):
    def __init__(self, src_path, des_path="all_sites.csv", start_pos=0, end_pos=20):
        self.src_path = src_path
        self.des_path = des_path
        self.start_pos = start_pos
        self.end_pos = end_pos
        self.keywords = []
        self._get_keywords_()
        self.extract_results()

    def _get_keywords_(self):
        if not os.path.isfile(self.src_path):
            print "No such file: %s" % self.src_path

        with codecs.open(self.src_path, 'r', encoding="utf-8") as f:
            for l in f.readlines():
                self.keywords.append(l.strip())

    def extract_results(self):
        print "Get all the sites..."
        all_results = []
        print "Extract search words:"
        for word in self.keywords:
            print word
            tmp = KeyWordSite(word, self.start_pos, self.end_pos)
            valid_results = KeyWordSite.get_processed_links(tmp.results)
            all_results.extend(valid_results)

        # Remove duplications
        single_result = []
        for result in all_results:
            if filter(lambda x: result[0] == x[0], single_result):
                pass
            else:
                single_result.append(result)
        with codecs.open(self.des_path, 'w') as f:
            map(lambda l: f.writelines(", ".join(l) + "\n"), single_result)