HaojuSpider

HaojuSpider.py

# -*- coding: utf-8 -*-
import scrapy
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor
from scrapy import Request

from telnum.items import TelnumItem

class HaojuSpider(CrawlSpider):
    name = "haoju"
    allowed_domains = ["shouji.dongganchaoren.com"]
    start_urls = (
        'http://shouji.dongganchaoren.com/',
    )

    rules = [
      Rule(LinkExtractor(allow=('/haoduan/[0-9]{3,3}/[0-9]{4,4}', )), callback='parse_hd')
    ]

    def parse_hd(self, response):
        s = response.xpath('//table/tr/td[1]/a')
        items = []
        for dd in s:
          it = TelnumItem()
          it['code'] = s.xpath('./text()').extract()
          it['url'] = s.xpath('./@href')[0].extract()
          items.append(it)
        return items