Download all links from a catalog.

links.py

import os
import json
import requests
from bs4 import BeautifulSoup

HOST = "http://data.dc.gov/"
ROOT = os.path.normpath(os.path.join(os.path.dirname(__file__), '../fixtures/'))
CATALOG = os.path.join(ROOT, 'catalog.html')
HTDOCS = os.path.join(ROOT, 'htdocs')

with open(CATALOG, 'r') as catalog:
    soup = BeautifulSoup(catalog, 'lxml')
    div = soup.find(id='ctl00_cphMain_linksDiv')
    downloads = []
    count = 0
    for a in div.find_all('a'):
        link = HOST + a['href']
        date = a.text
        response = requests.get(link, stream=True)
        if response.status_code == 200:
            count += 1
            path = os.path.join(HTDOCS, 'download%i.csv' % count)
            with open(path, 'w') as f:
                for chunk in response.iter_content(4096):
                    f.write(chunk)

            downloads.append({
                'date_range': date,
                'download':path
            }

with open('manifest.json', 'w') as manifest:
    json.dump(downloads, manifest, indent=4)