prettifythis

gistfile1.py

class ScruffySpider(BaseSpider):
    name = "scruffy" 
    allowed_domains = ['constantcontact.com']
    start_urls = [
        'http://www.constantcontact.com/index.jsp'
    ]

    domain_pattern = re.compile('constantcontact\.com')
    mailto_pattern = re.compile('mailto\:')

    visited_pages = []

    def parse(self, response):

        hcs = HtmlCSSSelector(response)
        links = hcs.select('a')
        items = []
        for link in links:
            url = link.get('href')
            if url is not None:
                if url[0] == '/':
                    url = 'http://www.constantcontact.com' + url
            if url != '#' and url is not None and self.domain_pattern.search(url) != None and self.mailto_pattern.search(url) == None and url in self.visited_pages == False:
                item = ScruffyItem()
                item["url"] = url 
                items.append(item)
                print url
        return items