Parse and render eBird datasets using leaflet heatmaps and headless chrome.
These are just the crucial files, not a working implementation.
https://github.com/glynnbird/countriesgeojson/
5 - common name
6 - scientific name
25 - lat
26 - lng
tail -n +2 <dataset> | cut -f5 | sort | uniq -c
| import multiprocessing | |
| import subprocess | |
| import json | |
| with open('json/__all__.json', 'r') as index: | |
| birds = json.loads(index.read()) | |
| def screenshot(slug): | |
| subprocess.run([ | |
| 'google-chrome-unstable', | |
| '--headless', | |
| '--disable-gpu', | |
| '--hide-scrollbars', | |
| '--window-size=1920,1080', | |
| f'--screenshot=screenshots/{slug}.png', | |
| f'file:///html/{slug}.html' | |
| ]) | |
| print(slug) | |
| if __name__ == '__main__': | |
| pool = multiprocessing.Pool(4) | |
| pool.map(screenshot, birds['slugs']) |
| from jinja2 import Environment, FileSystemLoader | |
| import json | |
| env = Environment( | |
| loader=FileSystemLoader('templates'), | |
| ) | |
| template = env.get_template('map-species.html') | |
| with open('json/__all__.json', 'r') as index: | |
| birds = json.loads(index.read()) | |
| print("N:", birds['count']) | |
| with open('geo/za.json', 'r') as g: | |
| geo = g.read() | |
| for slug in birds['slugs']: | |
| with open(f'json/{slug}.json') as infile: | |
| data = json.loads(infile.read()) | |
| context = { | |
| **data, | |
| "geo": geo, | |
| } | |
| html = template.render(context) | |
| with open(f'html/{slug}.html', 'w') as outfile: | |
| outfile.write(html) |
| <!DOCTYPE html> | |
| <html lang="en"> | |
| <head> | |
| <meta charset="UTF-8"> | |
| <title>{{ common_name }} | {{ scientific_name }}</title> | |
| <link rel="stylesheet" href="../css/leaflet-120.css"> | |
| <style> | |
| body { | |
| margin: 0 | |
| } | |
| #meta { | |
| position: absolute; | |
| bottom: 6rem; | |
| left: 50%; | |
| transform: translateX(-50%); | |
| font-family: Arial, sans-serif; | |
| } | |
| #birdname { | |
| font-size: 3rem; | |
| } | |
| #scientific { | |
| font-size: 1.2rem; | |
| color: #333 | |
| } | |
| </style> | |
| </head> | |
| <body> | |
| <div style="position: relative"> | |
| <div id="map" style="height: 1080px;width: 100%"></div> | |
| <div id="meta"> | |
| <h1 id="birdname">{{ common_name }}</h1> | |
| <h3 id="scientific">{{ scientific_name }}</h3> | |
| </div> | |
| </div> | |
| <script src="../js/leaflet-120.js"></script> | |
| <script src="../js/leaflet-heat.js"></script> | |
| <script> | |
| const map = L.map('map').setView([-30, 25], 6); | |
| L.geoJSON({{ geo|safe }}, { | |
| style: function (feature) { | |
| return {color: '#fff'} | |
| } | |
| }).addTo(map) | |
| let radius = 40; | |
| let blur = 20; | |
| L.heatLayer({{ sightings|safe }}, { | |
| radius: radius, | |
| blur: blur, | |
| maxZoom: 17, | |
| minOpacity: 0.3 | |
| }).addTo(map) | |
| </script> | |
| </body> | |
| </html> |
| import json | |
| import csv | |
| import sys | |
| from collections import defaultdict | |
| import re | |
| def slugify(value): | |
| value = re.sub(r'[^\w\s-]', '', value).strip().lower() | |
| return re.sub(r'[-\s]+', '-', value) | |
| def bird_factory(): | |
| return { | |
| "scientific_name": "", | |
| "common_name": "", | |
| "sightings": [] | |
| } | |
| if __name__ == '__main__': | |
| filepath = sys.argv[1] | |
| with open(filepath, 'r') as datafile: | |
| birds = defaultdict(bird_factory) | |
| reader = csv.reader(datafile, delimiter="\t") | |
| next(reader) # headers | |
| for line in reader: | |
| common_name = line[4] | |
| scientific_name = line[5] | |
| lat = line[24] | |
| lng = line[25] | |
| slug = slugify(scientific_name) | |
| birds[slug]['common_name'] = common_name | |
| birds[slug]['scientific_name'] = scientific_name | |
| birds[slug]['sightings'].append({ | |
| 'lat': lat, | |
| 'lng': lng, | |
| }) | |
| print("Saving...") | |
| names = {slug for slug in birds.keys()} | |
| print(len(names)) | |
| with open('json/__all__.json', 'w') as index: | |
| index.write(json.dumps({ | |
| "count": len(names), | |
| "slugs": list(names), | |
| })) | |
| for slug, value in birds.items(): | |
| with open(f'json/{slug}.json', 'w') as out: | |
| out.write(json.dumps(value)) |