Created
December 26, 2024 13:42
-
-
Save bpevs/82f240f862310aaf41bb995f2db04118 to your computer and use it in GitHub Desktop.
Script for fetching a list of the regions geofabrik generates
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/** | |
* All of these regions should have a `.poly` and a `-latest.osm.pbf` file | |
* associated to them. | |
*/ | |
import * as cheerio from "npm:cheerio" | |
const baseURL = "https://download.geofabrik.de/" | |
const regions = new Set() | |
// antarctica has no children, so not in baseRegions, but add to regions | |
const baseRegions = [ | |
"australia-oceania", | |
"asia", | |
"africa", | |
"central-america", | |
"europe", | |
"north-america", | |
"south-america", | |
] | |
baseRegions.concat(["antarctica"]) | |
.forEach(region => regions.add(region)) | |
const responses = baseRegions.map(fetchRegion) | |
await Promise.all(responses) | |
const regionsArray = [...regions]; | |
Deno.writeTextFile('regions.txt', regionsArray.sort().join("\n") + "\n") | |
async function fetchRegion(region: string) { | |
console.log('fetching: ', region) | |
const url = baseURL + "/" + region | |
const resp = await fetch(url) | |
const html = await resp.text() | |
const { directories, polyFiles } = await parseGeofabrikDirectory(html) | |
polyFiles | |
.forEach(file => regions.add(region + "/" + file.replace(".poly", ""))) | |
const childrenResp = directories.map( | |
(name) => { | |
const childName = name.replace(/^\//, "").replace(/\/$/, "") | |
if (!regions.has(childName)) { | |
return fetchRegion(region + "/" + childName) | |
} | |
} | |
) | |
await Promise.all(childrenResp) | |
} | |
async function parseGeofabrikDirectory(html: string) { | |
const $ = cheerio.load(html) | |
const rows = $("tr") | |
const directories: string[] = [] | |
const polyFiles: string[] = [] | |
rows.each((_, row) => { | |
const href = $(row).find("td a").first().attr("href") | |
if (!href) return | |
if (href.endsWith("/") && (href !== "/") && !href.endsWith("-updates/")) { | |
directories.push(href) | |
} | |
else if (href.endsWith(".poly")) { | |
polyFiles.push(href.replace("/", "")) | |
} | |
}) | |
return { | |
directories, | |
polyFiles | |
} | |
} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
africa | |
africa/algeria | |
africa/angola | |
africa/benin | |
africa/botswana | |
africa/burkina-faso | |
africa/burundi | |
africa/cameroon | |
africa/canary-islands | |
africa/cape-verde | |
africa/central-african-republic | |
africa/chad | |
africa/comores | |
africa/congo-brazzaville | |
africa/congo-democratic-republic | |
africa/djibouti | |
africa/egypt | |
africa/equatorial-guinea | |
africa/eritrea | |
africa/ethiopia | |
africa/gabon | |
africa/ghana | |
africa/guinea | |
africa/guinea-bissau | |
africa/ivory-coast | |
africa/kenya | |
africa/lesotho | |
africa/liberia | |
africa/libya | |
africa/madagascar | |
africa/malawi | |
africa/mali | |
africa/mauritania | |
africa/mauritius | |
africa/morocco | |
africa/mozambique | |
africa/namibia | |
africa/niger | |
africa/nigeria | |
africa/rwanda | |
africa/saint-helena-ascension-and-tristan-da-cunha | |
africa/sao-tome-and-principe | |
africa/senegal-and-gambia | |
africa/seychelles | |
africa/sierra-leone | |
africa/somalia | |
africa/south-africa | |
africa/south-africa-and-lesotho | |
africa/south-sudan | |
africa/sudan | |
africa/swaziland | |
africa/tanzania | |
africa/togo | |
africa/tunisia | |
africa/uganda | |
africa/zambia | |
africa/zimbabwe | |
antarctica | |
asia | |
asia/afghanistan | |
asia/armenia | |
asia/azerbaijan | |
asia/bangladesh | |
asia/bhutan | |
asia/cambodia | |
asia/china | |
asia/china/anhui | |
asia/china/beijing | |
asia/china/chongqing | |
asia/china/fujian | |
asia/china/gansu | |
asia/china/guangdong | |
asia/china/guangxi | |
asia/china/guizhou | |
asia/china/hainan | |
asia/china/hebei | |
asia/china/heilongjiang | |
asia/china/henan | |
asia/china/hong-kong | |
asia/china/hubei | |
asia/china/hunan | |
asia/china/inner-mongolia | |
asia/china/jiangsu | |
asia/china/jiangxi | |
asia/china/jilin | |
asia/china/liaoning | |
asia/china/macau | |
asia/china/ningxia | |
asia/china/qinghai | |
asia/china/shaanxi | |
asia/china/shandong | |
asia/china/shanghai | |
asia/china/shanxi | |
asia/china/sichuan | |
asia/china/tianjin | |
asia/china/tibet | |
asia/china/xinjiang | |
asia/china/yunnan | |
asia/china/zhejiang | |
asia/east-timor | |
asia/gcc-states | |
asia/india | |
asia/india/central-zone | |
asia/india/eastern-zone | |
asia/india/north-eastern-zone | |
asia/india/northern-zone | |
asia/india/southern-zone | |
asia/india/western-zone | |
asia/indonesia | |
asia/indonesia/java | |
asia/indonesia/kalimantan | |
asia/indonesia/maluku | |
asia/indonesia/nusa-tenggara | |
asia/indonesia/papua | |
asia/indonesia/sulawesi | |
asia/indonesia/sumatra | |
asia/iran | |
asia/iraq | |
asia/israel-and-palestine | |
asia/japan | |
asia/japan/chubu | |
asia/japan/chugoku | |
asia/japan/hokkaido | |
asia/japan/kansai | |
asia/japan/kanto | |
asia/japan/kyushu | |
asia/japan/shikoku | |
asia/japan/tohoku | |
asia/jordan | |
asia/kazakhstan | |
asia/kyrgyzstan | |
asia/laos | |
asia/lebanon | |
asia/malaysia-singapore-brunei | |
asia/maldives | |
asia/mongolia | |
asia/myanmar | |
asia/nepal | |
asia/north-korea | |
asia/pakistan | |
asia/philippines | |
asia/russia-asian-part | |
asia/south-korea | |
asia/sri-lanka | |
asia/syria | |
asia/taiwan | |
asia/tajikistan | |
asia/thailand | |
asia/turkmenistan | |
asia/uzbekistan | |
asia/vietnam | |
asia/yemen | |
australia-oceania | |
australia-oceania/american-oceania | |
australia-oceania/australia | |
australia-oceania/cook-islands | |
australia-oceania/fiji | |
australia-oceania/ile-de-clipperton | |
australia-oceania/kiribati | |
australia-oceania/marshall-islands | |
australia-oceania/micronesia | |
australia-oceania/nauru | |
australia-oceania/new-caledonia | |
australia-oceania/new-zealand | |
australia-oceania/niue | |
australia-oceania/palau | |
australia-oceania/papua-new-guinea | |
australia-oceania/pitcairn-islands | |
australia-oceania/polynesie-francaise | |
australia-oceania/samoa | |
australia-oceania/solomon-islands | |
australia-oceania/tokelau | |
australia-oceania/tonga | |
australia-oceania/tuvalu | |
australia-oceania/vanuatu | |
australia-oceania/wallis-et-futuna | |
central-america | |
central-america/bahamas | |
central-america/belize | |
central-america/costa-rica | |
central-america/cuba | |
central-america/el-salvador | |
central-america/guatemala | |
central-america/haiti-and-domrep | |
central-america/honduras | |
central-america/jamaica | |
central-america/nicaragua | |
central-america/panama | |
europe | |
europe/albania | |
europe/alps | |
europe/andorra | |
europe/austria | |
europe/azores | |
europe/belarus | |
europe/belgium | |
europe/bosnia-herzegovina | |
europe/britain-and-ireland | |
europe/bulgaria | |
europe/croatia | |
europe/cyprus | |
europe/czech-republic | |
europe/dach | |
europe/denmark | |
europe/estonia | |
europe/faroe-islands | |
europe/finland | |
europe/france | |
europe/france/alsace | |
europe/france/aquitaine | |
europe/france/auvergne | |
europe/france/basse-normandie | |
europe/france/bourgogne | |
europe/france/bretagne | |
europe/france/centre | |
europe/france/champagne-ardenne | |
europe/france/corse | |
europe/france/franche-comte | |
europe/france/guadeloupe | |
europe/france/guyane | |
europe/france/haute-normandie | |
europe/france/ile-de-france | |
europe/france/languedoc-roussillon | |
europe/france/limousin | |
europe/france/lorraine | |
europe/france/martinique | |
europe/france/mayotte | |
europe/france/midi-pyrenees | |
europe/france/nord-pas-de-calais | |
europe/france/pays-de-la-loire | |
europe/france/picardie | |
europe/france/poitou-charentes | |
europe/france/provence-alpes-cote-d-azur | |
europe/france/reunion | |
europe/france/rhone-alpes | |
europe/georgia | |
europe/germany | |
europe/germany/baden-wuerttemberg | |
europe/germany/baden-wuerttemberg/freiburg-regbez | |
europe/germany/baden-wuerttemberg/karlsruhe-regbez | |
europe/germany/baden-wuerttemberg/stuttgart-regbez | |
europe/germany/baden-wuerttemberg/tuebingen-regbez | |
europe/germany/bayern | |
europe/germany/bayern/mittelfranken | |
europe/germany/bayern/niederbayern | |
europe/germany/bayern/oberbayern | |
europe/germany/bayern/oberfranken | |
europe/germany/bayern/oberpfalz | |
europe/germany/bayern/schwaben | |
europe/germany/bayern/unterfranken | |
europe/germany/berlin | |
europe/germany/brandenburg | |
europe/germany/bremen | |
europe/germany/hamburg | |
europe/germany/hessen | |
europe/germany/mecklenburg-vorpommern | |
europe/germany/niedersachsen | |
europe/germany/nordrhein-westfalen | |
europe/germany/nordrhein-westfalen/arnsberg-regbez | |
europe/germany/nordrhein-westfalen/detmold-regbez | |
europe/germany/nordrhein-westfalen/duesseldorf-regbez | |
europe/germany/nordrhein-westfalen/koeln-regbez | |
europe/germany/nordrhein-westfalen/muenster-regbez | |
europe/germany/rheinland-pfalz | |
europe/germany/saarland | |
europe/germany/sachsen | |
europe/germany/sachsen-anhalt | |
europe/germany/schleswig-holstein | |
europe/germany/thueringen | |
europe/great-britain | |
europe/great-britain/united-kingdom | |
europe/greece | |
europe/guernsey-jersey | |
europe/hungary | |
europe/iceland | |
europe/ireland-and-northern-ireland | |
europe/isle-of-man | |
europe/italy | |
europe/italy/centro | |
europe/italy/isole | |
europe/italy/nord-est | |
europe/italy/nord-ovest | |
europe/italy/sud | |
europe/kosovo | |
europe/latvia | |
europe/liechtenstein | |
europe/lithuania | |
europe/luxembourg | |
europe/macedonia | |
europe/malta | |
europe/moldova | |
europe/monaco | |
europe/montenegro | |
europe/netherlands | |
europe/netherlands/drenthe | |
europe/netherlands/flevoland | |
europe/netherlands/friesland | |
europe/netherlands/gelderland | |
europe/netherlands/groningen | |
europe/netherlands/limburg | |
europe/netherlands/noord-brabant | |
europe/netherlands/noord-holland | |
europe/netherlands/overijssel | |
europe/netherlands/utrecht | |
europe/netherlands/zeeland | |
europe/netherlands/zuid-holland | |
europe/norway | |
europe/poland | |
europe/poland/dolnoslaskie | |
europe/poland/kujawsko-pomorskie | |
europe/poland/lodzkie | |
europe/poland/lubelskie | |
europe/poland/lubuskie | |
europe/poland/malopolskie | |
europe/poland/mazowieckie | |
europe/poland/opolskie | |
europe/poland/podkarpackie | |
europe/poland/podlaskie | |
europe/poland/pomorskie | |
europe/poland/slaskie | |
europe/poland/swietokrzyskie | |
europe/poland/warminsko-mazurskie | |
europe/poland/wielkopolskie | |
europe/poland/zachodniopomorskie | |
europe/portugal | |
europe/romania | |
europe/serbia | |
europe/slovakia | |
europe/slovenia | |
europe/spain | |
europe/spain/andalucia | |
europe/spain/aragon | |
europe/spain/asturias | |
europe/spain/cantabria | |
europe/spain/castilla-la-mancha | |
europe/spain/castilla-y-leon | |
europe/spain/cataluna | |
europe/spain/ceuta | |
europe/spain/extremadura | |
europe/spain/galicia | |
europe/spain/islas-baleares | |
europe/spain/la-rioja | |
europe/spain/madrid | |
europe/spain/melilla | |
europe/spain/murcia | |
europe/spain/navarra | |
europe/spain/pais-vasco | |
europe/spain/valencia | |
europe/sweden | |
europe/switzerland | |
europe/turkey | |
europe/ukraine | |
europe/united-kingdom | |
europe/united-kingdom/england | |
europe/united-kingdom/england/bedfordshire | |
europe/united-kingdom/england/berkshire | |
europe/united-kingdom/england/bristol | |
europe/united-kingdom/england/buckinghamshire | |
europe/united-kingdom/england/cambridgeshire | |
europe/united-kingdom/england/cheshire | |
europe/united-kingdom/england/cornwall | |
europe/united-kingdom/england/cumbria | |
europe/united-kingdom/england/derbyshire | |
europe/united-kingdom/england/devon | |
europe/united-kingdom/england/dorset | |
europe/united-kingdom/england/durham | |
europe/united-kingdom/england/east-sussex | |
europe/united-kingdom/england/east-yorkshire-with-hull | |
europe/united-kingdom/england/essex | |
europe/united-kingdom/england/gloucestershire | |
europe/united-kingdom/england/greater-london | |
europe/united-kingdom/england/greater-manchester | |
europe/united-kingdom/england/hampshire | |
europe/united-kingdom/england/herefordshire | |
europe/united-kingdom/england/hertfordshire | |
europe/united-kingdom/england/isle-of-wight | |
europe/united-kingdom/england/kent | |
europe/united-kingdom/england/lancashire | |
europe/united-kingdom/england/leicestershire | |
europe/united-kingdom/england/lincolnshire | |
europe/united-kingdom/england/london/enfield | |
europe/united-kingdom/england/merseyside | |
europe/united-kingdom/england/norfolk | |
europe/united-kingdom/england/north-yorkshire | |
europe/united-kingdom/england/northamptonshire | |
europe/united-kingdom/england/northumberland | |
europe/united-kingdom/england/nottinghamshire | |
europe/united-kingdom/england/oxfordshire | |
europe/united-kingdom/england/rutland | |
europe/united-kingdom/england/shropshire | |
europe/united-kingdom/england/somerset | |
europe/united-kingdom/england/south-yorkshire | |
europe/united-kingdom/england/staffordshire | |
europe/united-kingdom/england/suffolk | |
europe/united-kingdom/england/surrey | |
europe/united-kingdom/england/tyne-and-wear | |
europe/united-kingdom/england/warwickshire | |
europe/united-kingdom/england/west-midlands | |
europe/united-kingdom/england/west-sussex | |
europe/united-kingdom/england/west-yorkshire | |
europe/united-kingdom/england/wiltshire | |
europe/united-kingdom/england/worcestershire | |
europe/united-kingdom/scotland | |
europe/united-kingdom/wales | |
north-america | |
north-america/canada | |
north-america/canada/alberta | |
north-america/canada/british-columbia | |
north-america/canada/manitoba | |
north-america/canada/new-brunswick | |
north-america/canada/newfoundland-and-labrador | |
north-america/canada/northwest-territories | |
north-america/canada/nova-scotia | |
north-america/canada/nunavut | |
north-america/canada/ontario | |
north-america/canada/prince-edward-island | |
north-america/canada/quebec | |
north-america/canada/saskatchewan | |
north-america/canada/yukon | |
north-america/greenland | |
north-america/mexico | |
north-america/us | |
north-america/us-midwest | |
north-america/us-northeast | |
north-america/us-pacific | |
north-america/us-south | |
north-america/us-west | |
north-america/us/alabama | |
north-america/us/alaska | |
north-america/us/arizona | |
north-america/us/arkansas | |
north-america/us/california | |
north-america/us/california/norcal | |
north-america/us/california/socal | |
north-america/us/colorado | |
north-america/us/connecticut | |
north-america/us/delaware | |
north-america/us/district-of-columbia | |
north-america/us/florida | |
north-america/us/georgia | |
north-america/us/hawaii | |
north-america/us/idaho | |
north-america/us/illinois | |
north-america/us/indiana | |
north-america/us/iowa | |
north-america/us/kansas | |
north-america/us/kentucky | |
north-america/us/louisiana | |
north-america/us/maine | |
north-america/us/maryland | |
north-america/us/massachusetts | |
north-america/us/michigan | |
north-america/us/minnesota | |
north-america/us/mississippi | |
north-america/us/missouri | |
north-america/us/montana | |
north-america/us/nebraska | |
north-america/us/nevada | |
north-america/us/new-hampshire | |
north-america/us/new-jersey | |
north-america/us/new-mexico | |
north-america/us/new-york | |
north-america/us/north-carolina | |
north-america/us/north-dakota | |
north-america/us/ohio | |
north-america/us/oklahoma | |
north-america/us/oregon | |
north-america/us/pennsylvania | |
north-america/us/puerto-rico | |
north-america/us/rhode-island | |
north-america/us/south-carolina | |
north-america/us/south-dakota | |
north-america/us/tennessee | |
north-america/us/texas | |
north-america/us/us-virgin-islands | |
north-america/us/utah | |
north-america/us/vermont | |
north-america/us/virginia | |
north-america/us/washington | |
north-america/us/west-virginia | |
north-america/us/wisconsin | |
north-america/us/wyoming | |
south-america | |
south-america/argentina | |
south-america/bolivia | |
south-america/brazil | |
south-america/brazil/centro-oeste | |
south-america/brazil/nordeste | |
south-america/brazil/norte | |
south-america/brazil/sudeste | |
south-america/brazil/sul | |
south-america/chile | |
south-america/colombia | |
south-america/ecuador | |
south-america/guyana | |
south-america/paraguay | |
south-america/peru | |
south-america/suriname | |
south-america/uruguay | |
south-america/venezuela |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment