commoncrawl_experiment.py

commoncrawl_experiment.py

# adapted (mostly stolen) from: 
# https://www.cedar.net.au/using-python-and-common-crawl-to-find-products-from-amazon-com/ 
# with conversion to python 3 + tweaks

import requests, json, io, gzip
from bs4 import BeautifulSoup

index_list = ["2018-26"]

### -----------------------
### Searches the Common Crawl Index for a domain.
### -----------------------
def search_domain(domain):
    record_list = []
#    print("[*] Trying target domain: %s" % domain)
    
    for index in index_list:
#        print("[*] Trying index %s" % index)
        cc_url  = "http://index.commoncrawl.org/CC-MAIN-%s-index?" % index
        cc_url += "url=%s&matchType=domain&output=json" % domain
        
        response = requests.get(cc_url)
#        print(response)
        
        if response.status_code == 200:
            records = response.content.splitlines()
            for record in records:
                record_list.append(json.loads(record))  
#            print("[*] Added %d results." % len(records))
#    print("[*] Found a total of %d hits." % len(record_list))
    return record_list

pg = search_domain("paul-gowder.com")
# rulelaw = search_domain("rulelaw.net")
#print(rulelaw)

def download_page(record):
 
    offset, length = int(record['offset']), int(record['length'])
    offset_end = offset + length - 1
 
    # We'll get the file via HTTPS so we don't need to worry about S3 credentials
    # Getting the file on S3 is equivalent however - you can request a Range
    prefix = 'https://commoncrawl.s3.amazonaws.com/'
    filename = record['filename']
    # print(prefix + filename)
 
    # We can then use the Range header to ask for just this set of bytes
    resp = requests.get(prefix + filename, headers={'Range': 'bytes={}-{}'.format(offset, offset_end)})
    #print(resp.content)
    #print(resp.text)
 
    # The page is stored compressed (gzip) to save space
    # We can extract it using the GZIP library
    raw_data = io.BytesIO(resp.content)
    f = gzip.GzipFile(fileobj=raw_data)
 
    # What we have now is just the WARC response, formatted:
    data = f.read()
    # get the html bit
    return data.decode('utf-8').strip().split("\r\n\r\n")[-1]
    

#for x in rulelaw:
#    print(download_page(x))

test_html = download_page(pg[-1])

# try https://github.com/internetarchive/warc to parse??

def extract_text(html):
    soup = BeautifulSoup(html)
    if soup is None:
        return None
    for crap in soup(["script", "style", "meta"]):
        crap.extract()
    return soup.get_text()

print(extract_text(test_html))