Last active
August 29, 2022 18:06
-
-
Save JossWhittle/53778ebaff015bed9a5caaa78b4648a3 to your computer and use it in GitHub Desktop.
Simple script to download all of the images on https://earthview.withgoogle.com/ in 1800x1200 resolution.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import os | |
import json | |
import urllib.request | |
from tqdm import tqdm | |
import multiprocessing | |
with urllib.request.urlopen('https://earthview.withgoogle.com/_api/photos.json') as f: | |
slugs = list(map(lambda slug: slug | dict(id=slug['slug'].split('-')[-1]), | |
json.loads(f.read().decode('utf-8')))) | |
with open('slug.json', 'w') as f: | |
json.dump(slugs, f) | |
def fetch_meta(slug): | |
path = 'meta/{slug}.json'.format(**slug) | |
if os.path.isfile(path): | |
# Load meta from disk | |
with open(path, 'r') as f: | |
meta = json.loads(f.read()) | |
else: | |
# Download meta | |
with urllib.request.urlopen('https://earthview.withgoogle.com/_api/{id}.json'.format(**slug)) as f: | |
meta = json.loads(f.read().decode('utf-8')) | |
# Save meta to disk | |
with open(path, 'w') as f: | |
json.dump(meta, f) | |
return meta | |
os.makedirs('./meta', exist_ok=True) | |
metas = list(multiprocessing.Pool(12).imap(fetch_meta, tqdm(slugs, desc='Fetch Meta'))) | |
with open('meta.json', 'w') as f: | |
json.dump(metas, f) | |
def fetch_image(meta): | |
path = 'images/{slug}.jpg'.format(**meta) | |
if not os.path.isfile(path): | |
# Download image to disk | |
_, headers = urllib.request.urlretrieve(meta['photoUrl'], path) | |
assert 'Content-Type: image/jpeg' in str(headers) | |
os.makedirs('./images', exist_ok=True) | |
multiprocessing.Pool(12).imap(fetch_image, tqdm(metas, desc='Fetch Images')) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment