Skip to content

Instantly share code, notes, and snippets.

@JossWhittle
Last active August 29, 2022 18:06
Show Gist options
  • Save JossWhittle/53778ebaff015bed9a5caaa78b4648a3 to your computer and use it in GitHub Desktop.
Save JossWhittle/53778ebaff015bed9a5caaa78b4648a3 to your computer and use it in GitHub Desktop.
Simple script to download all of the images on https://earthview.withgoogle.com/ in 1800x1200 resolution.
import os
import json
import urllib.request
from tqdm import tqdm
import multiprocessing
with urllib.request.urlopen('https://earthview.withgoogle.com/_api/photos.json') as f:
slugs = list(map(lambda slug: slug | dict(id=slug['slug'].split('-')[-1]),
json.loads(f.read().decode('utf-8'))))
with open('slug.json', 'w') as f:
json.dump(slugs, f)
def fetch_meta(slug):
path = 'meta/{slug}.json'.format(**slug)
if os.path.isfile(path):
# Load meta from disk
with open(path, 'r') as f:
meta = json.loads(f.read())
else:
# Download meta
with urllib.request.urlopen('https://earthview.withgoogle.com/_api/{id}.json'.format(**slug)) as f:
meta = json.loads(f.read().decode('utf-8'))
# Save meta to disk
with open(path, 'w') as f:
json.dump(meta, f)
return meta
os.makedirs('./meta', exist_ok=True)
metas = list(multiprocessing.Pool(12).imap(fetch_meta, tqdm(slugs, desc='Fetch Meta')))
with open('meta.json', 'w') as f:
json.dump(metas, f)
def fetch_image(meta):
path = 'images/{slug}.jpg'.format(**meta)
if not os.path.isfile(path):
# Download image to disk
_, headers = urllib.request.urlretrieve(meta['photoUrl'], path)
assert 'Content-Type: image/jpeg' in str(headers)
os.makedirs('./images', exist_ok=True)
multiprocessing.Pool(12).imap(fetch_image, tqdm(metas, desc='Fetch Images'))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment