batch download darksky timemachine api

darksky.py

import os
import time
from datetime import datetime, timedelta
from dotenv import load_dotenv, find_dotenv
from urllib.request import urlopen
from concurrent.futures import ThreadPoolExecutor, wait

def main():
    print("running!")
    load_dotenv(find_dotenv())
    date = datetime(2015, 1, 1)
    days = range(0, 500)
    dates = (date + timedelta(days=i) for i in days)
    with ThreadPoolExecutor(5) as executor:
        futures = [executor.submit(maybe_download, d) for d in dates]
        wait(futures)

def maybe_download(date: datetime) -> int:
    stamp = str(date).replace(' ', '_')
    cache = 'data/%s.json' % stamp

    if not os.path.exists(cache):
        print('Downloading file...')
        return download(date, cache)
    else:
        size = os.stat(cache).st_size
        print('found {file} with size {size}'.format(file=cache, size=size))
        return size

def download(date: datetime, cache: str) -> int:
    api = os.environ['DARKSKY_API_KEY']
    location = '37.8267,-122.4233'
    epoch = int(time.mktime(date.timetuple()))
    host = 'https://api.darksky.net'
    path = '/forecast/{key}/{loc},{time}?exclude=currently,flags'
    url = host + path.format(key=api, loc=location, time=epoch)

    print('downloading %s' % url)
    try:
        with urlopen(url) as u, open(cache, 'wb') as f:
            return f.write(u.read())
    except:
        print('Exception downloading or saving file %s %s' % url, cache)
        return 0


if __name__ == '__main__':
    main()