gistfile1.txt

    def fetch(self, url):
        logging.info('Fetching %s', url)

        urlparts = urlparse(url)
        conn = httplib.HTTPConnection(urlparts.hostname, urlparts.port, timeout=30)

        try:
            conn.request('GET', urlparts.path + '?' + urlparts.query)
            resp = conn.getresponse()
            self.results[url] = resp.read()
        except Exception:
            logging.error('Failed to fetch %s', url, exc_info=True)