Last active
September 2, 2017 02:45
-
-
Save aaronpdennis/a53f9afb8ef24d499e96 to your computer and use it in GitHub Desktop.
Process for inputing OpenStreetMap .pbf files, generating vector tiles based off a humanitarian data model, and then hosting those tiles as a data source on Mapbox.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"water_source": { | |
"class": { | |
"water well": ["man_made=water_well"], | |
"water tower": ["man_made=water_tower"], | |
"water tank": ["man_made=storage_tank", "man_made=water_tank"], | |
"spring": ["natural=spring"], | |
"drinking water": ["amenity=drinking_water"] | |
}, | |
"potable": { | |
"yes": ["drinking_water=yes", "amenity=drinking_water"], | |
"no": ["drinking_water=no"] | |
}, | |
"pump": { | |
"yes": ["pump=yes"], | |
"manual": ["pump=manual", "pump_type=manual"], | |
"powered": ["pump=powered", "pump_type=powered_network"] | |
} | |
}, | |
"medical": { | |
"class": { | |
"hospital": ["amenity=hospital"], | |
"clinic": ["amenity=clinic"], | |
"field hospital": ["health_facility_type=field_hospital", "health_facility:type=field_hospital"], | |
"pharmacy": ["amenity=pharmacy"], | |
"dispensary": ["health_facility_type=dispensary", "health_facility:type=dispensary"], | |
"morgue": ["amenity=mortuary"] | |
} | |
}, | |
"sanitation": { | |
"class": { | |
"shower": ["amenity=shower"], | |
"toilet": ["amenity=toilets"], | |
"waste": ["amenity=waste_disposal", "amenity=waste_basket", "landuse=landfill", "amenity=sanitary_dump_station", "man_made=wastewater_plant", "informal=dumpsite"] | |
} | |
}, | |
"communication": { | |
"class": { | |
"tower": ["tower:type=communication", "man_made=communications_tower"], | |
"dish": ["man_made=communications_dish"], | |
"notice board": ["amenity=community_information_kiosk", "board_type=notice"] | |
} | |
}, | |
"electric_utility": { | |
"class": { | |
"generator": ["power=generator"], | |
"transmission": ["power=tower", "power=pole", "power=line", "power=minor_line"], | |
"distribution": ["power=station", "power=substation", "power=sub_station", "power=transformer"] | |
}, | |
"power_source": { | |
"wind": ["generator:source=wind"], | |
"solar": ["generator:source=solar"], | |
"hydro": ["generator:source=hydro"], | |
"gas": ["generator:source=gas"], | |
"coal": ["generator:source=coal"], | |
"biomass": ["generator:source=biomass"], | |
"nuclear": ["generator:source=nuclear"] | |
}, | |
"structure": { | |
"tower": ["power=tower"], | |
"pole": ["power=pole"], | |
"line": ["power=line", "power=minor_line"] | |
} | |
}, | |
"emergency": { | |
"class": { | |
"medical rescue": ["emergency=ambulance_station", "emergency=defibrillator", "emergency=aed", "medical=aed"], | |
"fire fighter": ["emergency=fire_extinguisher", "emergency=fire_hydrant", "emergency=fire_hose", "emergency=fire_flapper"], | |
"lifeguarding": ["emergency=lifeguard_base", "emergency=lifeguard_platform", "emergency=lifeguard_place"], | |
"assembly point": ["emergency=assembly_point"], | |
"access point": ["emergency=access_point"], | |
"emergency phone": ["emergency=phone"], | |
"emergency siren": ["emergency=siren"], | |
"helicopter": ["aeroway=helipad", "emergency:helipad=potential", "emergency=landing_site"] | |
} | |
}, | |
"building_condition": { | |
"class": { | |
"construction": ["building=construction"], | |
"damaged": ["damage=moderate", "damage=extensive", "damage=severe"], | |
"collapsed": ["damage=total", "damage=destroyed", "building=collapsed", "building=collapse", "damage=partially_collapsed", "damage=collapsed"], | |
"flooded": ["damage=flooded"] | |
} | |
}, | |
"road_condition": { | |
"class": { | |
"surface": ["surface=unpaved", "smoothness=bad"], | |
"condition": ["condition=bad", "impassable=yes", "bridge=collapsed"], | |
"barrier": ["military=checkpoint", "barrier=checkpoint", "barrier=debris", "barrier=gate"] | |
}, | |
"surface": { | |
"unpaved": ["surface=unpaved"], | |
"rough": ["smoothness=bad"] | |
}, | |
"condition": { | |
"bad": ["condition=bad"], | |
"impassable": ["impassable=yes"], | |
"collapsed": ["bridge=collapsed"] | |
}, | |
"barrier": { | |
"checkpoint": ["military=checkpoint", "barrier=checkpoint"], | |
"debris": ["barrier=debris"], | |
"gate": ["barrier=gate"] | |
} | |
}, | |
"site" : { | |
"class": { | |
"shelter": ["amenity=shelter", "shelter=yes"], | |
"camp": ["refugee=yes", "idp:camp_site=spontaneous_camp"], | |
"residential": ["landuse=residential"], | |
"common": ["leisure=common"], | |
"rubble": ["landuse=brownfield"], | |
"landslide": ["hazard=landslide"] | |
} | |
} | |
} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env node | |
var fs = require('fs'); | |
var osmium = require('osmium'); | |
var turf = require('turf'); | |
var location_handler = new osmium.LocationHandler(); | |
var handler = new osmium.Handler(); | |
var hdm = JSON.parse(fs.readFileSync('hdm.json', 'utf8')); | |
var hdmTags = []; | |
var hdmClasses = []; | |
var hdmLayers = []; | |
var tagClassAndLayer = {}; | |
var hdmClassLayer = {}; | |
function findPropertyFromTag(tag, layer) { | |
for (var property in hdm[layer]) { | |
if (hdm[layer].hasOwnProperty(property) && property !== "class") { | |
for (value in hdm[layer][property]) { | |
if (hdm[layer][property].hasOwnProperty(value)) { | |
for (var i = 0; i < hdm[layer][property][value].length; i++) { | |
if (hdm[layer][property][value].indexOf(tag) > -1) { | |
return { "property" : property, "value" : value }; | |
} | |
} | |
} | |
} | |
} | |
} | |
} | |
for (var layer in hdm) { | |
if (hdm.hasOwnProperty(layer)) { | |
hdmLayers.push(layer); | |
for (classAttr in hdm[layer]["class"]) { | |
if (hdm[layer]["class"].hasOwnProperty(classAttr)) { | |
hdmClasses.push(classAttr); | |
for (var i = 0; i < hdm[layer]["class"][classAttr].length; i++) { | |
var osmTag = hdm[layer]["class"][classAttr][i]; | |
hdmTags.push(osmTag); | |
tagClassAndLayer[osmTag] = { "class": classAttr, "layer": layer }; | |
} | |
} | |
} | |
} | |
} | |
handler.options({ 'tagged_nodes_only' : true }); | |
handler.on('node', filter); | |
handler.on('way', filter); | |
var dir = './hdm-data'; | |
if (!fs.existsSync(dir)){ | |
fs.mkdirSync(dir); | |
} | |
var wstreams = {}; | |
for (var i = 0; i < hdmLayers.length; i++) { | |
wstreams[hdmLayers[i]] = fs.createWriteStream('hdm-data/' + hdmLayers[i] + '.json'); | |
} | |
var labelWStream = fs.createWriteStream('hdm-data/hdm_label.json'); | |
var counter = 0; | |
function filter(item) { | |
var tags = item.tags(); | |
var keys = Object.keys(tags); | |
keys.forEach(function(key) { | |
var candidate = key + '=' + tags[key]; | |
if ((hdmTags.indexOf(candidate) > -1)) { | |
counter++; | |
var layer = tagClassAndLayer[candidate].layer; | |
var geometry = item.geojson(); | |
var properties = {}; | |
properties.class = tagClassAndLayer[candidate].class; | |
properties.tag = candidate; | |
if (findPropertyFromTag(candidate, layer)) { | |
var otherProperties = findPropertyFromTag(candidate, layer); | |
properties[otherProperties.property] = otherProperties.value; | |
} | |
if (keys.indexOf('name' > -1)) { | |
properties['name'] = tags['name']; | |
} | |
if (item.coordinates !== undefined) { | |
properties.geom = 'Point'; | |
properties.osm_id = parseInt(item.id) + Math.pow(10, 15); | |
} else if ( | |
item.geojson().coordinates[0][0] === item.geojson().coordinates[item.geojson().coordinates.length - 1][0] && | |
item.geojson().coordinates[0][1] === item.geojson().coordinates[item.geojson().coordinates.length - 1][1] | |
) { | |
properties.geom = 'Polygon'; | |
properties.osm_id = parseInt(item.id) + Math.pow(10, 12); | |
} else { | |
properties.geom = 'LineString'; | |
properties.osm_id = item.id; | |
} | |
wstreams[layer].write( | |
JSON.stringify({ | |
'type': 'Feature', | |
'properties': properties, | |
'geometry': item.geojson() | |
}) + "\n" | |
); | |
if ( properties.class !== 'residential' | |
&& properties.class !== 'common' | |
&& properties.class !== 'rubble' | |
&& properties.class !== 'landslide' | |
&& properties.class !== 'transmission' | |
&& layer !== 'road_condition') | |
{ | |
var feature = { | |
'type': 'Feature', | |
'properties': {}, | |
'geometry': { | |
'type': properties.geom, | |
'coordinates': properties.geom !== 'Polygon' ? item.geojson().coordinates : [item.geojson().coordinates] | |
} | |
}; | |
var labelPt = turf.pointOnSurface(feature); | |
labelPt.properties = properties; | |
labelPt.properties.layer = layer; | |
labelPt.properties.geom = 'Point'; | |
labelWStream.write(JSON.stringify(labelPt) + '\n'); | |
} | |
process.stdout.write('feature count: ' + counter + '\r'); | |
} | |
}); | |
} | |
function logFeatureCount() { | |
process.stdout.write('feature count: ' + counter + '\n\n'); | |
counter = 0; | |
} | |
for (var i = 2; i < process.argv.length; i++) { | |
process.stdout.write(process.argv[i] + '\n'); | |
file = new osmium.File(process.argv[i], 'pbf'); | |
reader = new osmium.Reader(file, { | |
node: true, | |
way: true | |
}); | |
osmium.apply(reader, location_handler, handler); | |
logFeatureCount(); | |
} |
hi, if i wish only to create large file dumps of GeoJSON features fot this cuba-latest.osm.pbf, and then use typpecanoe to generate de .mbtiles, is mandatory to use hdm.json, what are the functions of *.json? where are they in the map?
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
For Ubuntu 14.04 (Trusty), if you want to use install osmium through npm you first need to install the right dependencies
...
sudo apt-add-repository --yes ppa:chris-lea/node.js
sudo apt-add-repository --yes ppa:ubuntu-toolchain-r/test
sudo apt-get -y update
sudo apt-get -y install git gcc-4.8 g++-4.8 build-essential nodejs
sudo apt-get -y install libboost-dev zlib1g-dev protobuf-compiler
sudo apt-get -y install libprotobuf-dev libexpat1-dev
sudo apt-get -y install libsparsehash-dev
export CC=gcc-4.8
export CXX=g++-4.8
git clone https://github.com/scrosby/OSM-binary.git
cd OSM-binary/src
make && sudo make install
...