recursive crawling

gistfile1.txt

import re
import mechanize
#from bs4 import BeautifulSoup

class Crawler:
    def __init__(self):
        self.url_list = []
        self.loop = 0

    def get_url_regex(self, url):
        return url + '(.*)/'

    def get_links(self, url):
        url_list = []
        br = mechanize.Browser()
        br.open(url)
        for link in br.links(url_regex=self.get_url_regex(url)):
            url_list.append(link.url)

        return url_list

    def crawl(self, url):
        self.loop = self.loop + 1
        links = self.get_links(url)
        self.url_list = self.url_list + links
        print self.loop

        for link in self.get_links(url):
            if link not in self.url_list:
                self.crawl(link)

foo = Crawler()
foo.crawl("http://www.epocacosmeticos.com.br/")
print foo.url_list