Process for inputing OpenStreetMap .pbf files, generating vector tiles based off a humanitarian data model, and then hosting those tiles as a data source on Mapbox.

hdm.json

{
  "water_source": {
    "class": {
      "water well": ["man_made=water_well"],
      "water tower": ["man_made=water_tower"],
      "water tank": ["man_made=storage_tank", "man_made=water_tank"],
      "spring": ["natural=spring"],
      "drinking water": ["amenity=drinking_water"]
    },
    "potable": {
      "yes": ["drinking_water=yes", "amenity=drinking_water"],
      "no": ["drinking_water=no"]
    },
    "pump": {
      "yes": ["pump=yes"],
      "manual": ["pump=manual", "pump_type=manual"],
      "powered": ["pump=powered", "pump_type=powered_network"]
    }
  },
  "medical": {
    "class": {
      "hospital": ["amenity=hospital"],
      "clinic": ["amenity=clinic"],
      "field hospital": ["health_facility_type=field_hospital", "health_facility:type=field_hospital"],
      "pharmacy": ["amenity=pharmacy"],
      "dispensary": ["health_facility_type=dispensary", "health_facility:type=dispensary"],
      "morgue": ["amenity=mortuary"]
    }
  },
  "sanitation": {
    "class": {
      "shower": ["amenity=shower"],
      "toilet": ["amenity=toilets"],
      "waste": ["amenity=waste_disposal", "amenity=waste_basket", "landuse=landfill", "amenity=sanitary_dump_station", "man_made=wastewater_plant", "informal=dumpsite"]
    }
  },
  "communication": {
    "class": {
      "tower": ["tower:type=communication", "man_made=communications_tower"],
      "dish": ["man_made=communications_dish"],
      "notice board": ["amenity=community_information_kiosk", "board_type=notice"]
    }
  },
  "electric_utility": {
    "class": {
      "generator": ["power=generator"],
      "transmission": ["power=tower", "power=pole", "power=line", "power=minor_line"],
      "distribution": ["power=station", "power=substation", "power=sub_station", "power=transformer"]
    },
    "power_source": {
      "wind": ["generator:source=wind"],
      "solar": ["generator:source=solar"],
      "hydro": ["generator:source=hydro"],
      "gas": ["generator:source=gas"],
      "coal": ["generator:source=coal"],
      "biomass": ["generator:source=biomass"],
      "nuclear": ["generator:source=nuclear"]
    },
    "structure": {
      "tower": ["power=tower"],
      "pole": ["power=pole"],
      "line": ["power=line", "power=minor_line"]
    }
  },
  "emergency": {
    "class": {
      "medical rescue": ["emergency=ambulance_station", "emergency=defibrillator", "emergency=aed", "medical=aed"],
      "fire fighter": ["emergency=fire_extinguisher", "emergency=fire_hydrant", "emergency=fire_hose", "emergency=fire_flapper"],
      "lifeguarding": ["emergency=lifeguard_base", "emergency=lifeguard_platform", "emergency=lifeguard_place"],
      "assembly point": ["emergency=assembly_point"],
      "access point": ["emergency=access_point"],
      "emergency phone": ["emergency=phone"],
      "emergency siren": ["emergency=siren"],
      "helicopter": ["aeroway=helipad", "emergency:helipad=potential", "emergency=landing_site"]
    }
  },
  "building_condition": {
    "class": {
      "construction": ["building=construction"],
      "damaged": ["damage=moderate", "damage=extensive", "damage=severe"],
      "collapsed": ["damage=total", "damage=destroyed", "building=collapsed", "building=collapse", "damage=partially_collapsed", "damage=collapsed"],
      "flooded": ["damage=flooded"]
    }
  },
  "road_condition": {
    "class": {
      "surface": ["surface=unpaved", "smoothness=bad"],
      "condition": ["condition=bad", "impassable=yes", "bridge=collapsed"],
      "barrier": ["military=checkpoint", "barrier=checkpoint", "barrier=debris", "barrier=gate"]
    },
    "surface": {
      "unpaved": ["surface=unpaved"],
      "rough": ["smoothness=bad"]
    },
    "condition": {
      "bad": ["condition=bad"],
      "impassable": ["impassable=yes"],
      "collapsed": ["bridge=collapsed"]
    },
    "barrier": {
      "checkpoint": ["military=checkpoint", "barrier=checkpoint"],
      "debris": ["barrier=debris"],
      "gate": ["barrier=gate"]
    }
  },
  "site" : {
    "class": {
      "shelter": ["amenity=shelter", "shelter=yes"],
      "camp": ["refugee=yes", "idp:camp_site=spontaneous_camp"],
      "residential": ["landuse=residential"],
      "common": ["leisure=common"],
      "rubble": ["landuse=brownfield"],
      "landslide": ["hazard=landslide"]
    }
  }
}

process.js

#!/usr/bin/env node

var fs = require('fs');
var osmium = require('osmium');
var turf = require('turf');

var location_handler = new osmium.LocationHandler();
var handler = new osmium.Handler();

var hdm = JSON.parse(fs.readFileSync('hdm.json', 'utf8'));

var hdmTags = [];
var hdmClasses = [];
var hdmLayers = [];

var tagClassAndLayer = {};
var hdmClassLayer = {};

function findPropertyFromTag(tag, layer) {
  for (var property in hdm[layer]) {
    if (hdm[layer].hasOwnProperty(property) && property !== "class") {
      for (value in hdm[layer][property]) {
        if (hdm[layer][property].hasOwnProperty(value)) {
          for (var i = 0; i < hdm[layer][property][value].length; i++) {
            if (hdm[layer][property][value].indexOf(tag) > -1) {
              return { "property" : property, "value" : value };
            }
          }
        }
      }
    }
  }
}

for (var layer in hdm) {
  if (hdm.hasOwnProperty(layer)) {
    hdmLayers.push(layer);
    for (classAttr in hdm[layer]["class"]) {
      if (hdm[layer]["class"].hasOwnProperty(classAttr)) {
        hdmClasses.push(classAttr);
        for (var i = 0; i < hdm[layer]["class"][classAttr].length; i++) {
          var osmTag = hdm[layer]["class"][classAttr][i];
          hdmTags.push(osmTag);
          tagClassAndLayer[osmTag] = { "class": classAttr, "layer": layer };
        }
      }
    }
  }
}

handler.options({ 'tagged_nodes_only' : true });

handler.on('node', filter);
handler.on('way', filter);

var dir = './hdm-data';
if (!fs.existsSync(dir)){
    fs.mkdirSync(dir);
}

var wstreams = {};
for (var i = 0; i < hdmLayers.length; i++) {
  wstreams[hdmLayers[i]] = fs.createWriteStream('hdm-data/' + hdmLayers[i] + '.json');
}

var labelWStream = fs.createWriteStream('hdm-data/hdm_label.json');

var counter = 0;

function filter(item) {
  var tags = item.tags();
  var keys = Object.keys(tags);
  keys.forEach(function(key) {
    var candidate = key + '=' + tags[key];

    if ((hdmTags.indexOf(candidate) > -1)) {

      counter++;

      var layer = tagClassAndLayer[candidate].layer;
      var geometry = item.geojson();

      var properties = {};
      properties.class = tagClassAndLayer[candidate].class;

      properties.tag = candidate;

      if (findPropertyFromTag(candidate, layer)) {
        var otherProperties = findPropertyFromTag(candidate, layer);
        properties[otherProperties.property] = otherProperties.value;
      }

      if (keys.indexOf('name' > -1)) {
        properties['name'] = tags['name'];
      }

      if (item.coordinates !== undefined) {
        properties.geom = 'Point';
        properties.osm_id = parseInt(item.id) + Math.pow(10, 15);
      } else if (
        item.geojson().coordinates[0][0] === item.geojson().coordinates[item.geojson().coordinates.length - 1][0] &&
        item.geojson().coordinates[0][1] === item.geojson().coordinates[item.geojson().coordinates.length - 1][1]
      ) {
        properties.geom = 'Polygon';
        properties.osm_id = parseInt(item.id) + Math.pow(10, 12);
      } else {
        properties.geom = 'LineString';
        properties.osm_id = item.id;
      }

      wstreams[layer].write(
        JSON.stringify({
          'type': 'Feature',
          'properties': properties,
          'geometry': item.geojson()
        }) + "\n"
      );

      if (   properties.class !== 'residential'
          && properties.class !== 'common'
          && properties.class !== 'rubble'
          && properties.class !== 'landslide'
          && properties.class !== 'transmission'
          && layer !== 'road_condition')
      {

        var feature = {
          'type': 'Feature',
          'properties': {},
          'geometry': {
            'type': properties.geom,
            'coordinates': properties.geom !== 'Polygon' ? item.geojson().coordinates : [item.geojson().coordinates]
          }
        };

        var labelPt = turf.pointOnSurface(feature);

        labelPt.properties = properties;
        labelPt.properties.layer = layer;
        labelPt.properties.geom = 'Point';
        labelWStream.write(JSON.stringify(labelPt) + '\n');
      }

      process.stdout.write('feature count: ' + counter + '\r');
    }
  });
}

function logFeatureCount() {
  process.stdout.write('feature count: ' + counter + '\n\n');
  counter = 0;
}

for (var i = 2; i < process.argv.length; i++) {

  process.stdout.write(process.argv[i] + '\n');

  file = new osmium.File(process.argv[i], 'pbf');
  reader = new osmium.Reader(file, {
      node: true,
      way: true
  });

  osmium.apply(reader, location_handler, handler);

  logFeatureCount();

}

Dependencies for Mac OS X:

• node version 010 : $ brew install https://raw.githubusercontent.com/Homebrew/homebrew-versions/master/node010.rb
• fs : $ npm install fs
• node-osmium : $ npm install osmium
• turf : $ npm install turf

Example:

$ ./process.js kathmandu_nepal.osm.pbf

If you're told the file doesn't have the right permissions first run $ chmod u+rwx ./process.js and then the command.

Generating Vector MBTiles

After running ./process.js, if you want to create a vector MBTiles file, install tippecanoe with

brew install tippecanoe

then do

tippecanoe \
  -z 14 -Z 13 \
  -d12 -D12 \
  -b 20 \
  -pc -f \
  -o hdm.mbtiles \
    hdm-data/hdm_label.json\
    hdm-data/water_source.json \
    hdm-data/communication.json \
    hdm-data/electric_utility.json \
    hdm-data/sanitation.json \
    hdm-data/emergency.json \
    hdm-data/medical.json \
    hdm-data/building_condition.json \
    hdm-data/road_condition.json \
    hdm-data/site.json

You can adjust the hdm.json file to create different data models for your vector tiles. In the Tippecanoe command above, each hdm-data/file.json is a layer that gets added to the MBTiles. If you create different layers in hdm.json, you'll have to update the file names in the Tippecanoe command above.

Uploading to Mapbox

You can manually upload the output hdm.mbtiles file from your Tippecanoe command to Mapbox's data hosting services here: https://www.mapbox.com/uploads/?source=data

You can also use the script below to upload the hdm.mbtiles file. You'll need to

1. Install the mapbox-upload dependency : $ npm install mapbox --save mapbox-upload
2. Change the third line, adding in your own Mapbox access token with an uploads:write scope. Create one of those tokens at https://www.mapbox.com/account/apps/
3. Change the 4th line to your Mapox user account

#!/usr/bin/env node

var mapboxAccessToken = '<your-mapbox-token-with-uploads:write-scope>'; // create a token here --> https://www.mapbox.com/account/apps/
var mapboxUserAccount = '<your-mapbox-account-user-name-here>';

var upload = require('mapbox-upload');

console.log('Starting upload.');

var progress = upload({
    file: __dirname + '/hdm.mbtiles',
    account: mapboxUserAccount,
    accesstoken: mapboxAccessToken,
    mapid: mapboxUserAccount + '.' + 'humanitarian-data-model'
});

progress.on('error', function(err){
    if (err) throw err;
});

progress.on('progress', function(p){
  process.stdout.write(Math.round(p.percentage,4) + "%\r");
})

progress.once('finished', function(){
    console.log("Tiles are processing at Mapbox.com");
});

Save this script above, updated with your access token, into a file upload.js, then run $ chmod u+rwx ./upload.js and then $ ./upload.js.

You can change the mapid, the identifier where your vector tiles are uploaded to, by switching out where it says 'humanitarian-data-model' on line 9 for some other valid mapid (like a short string of text, no spaces).

Using the Mapbox Studio Humanitarian Style

I've designed a Mapbox Studio style around these humanitarian data model vector tiles. It looks like this.

To use this style for your projects, download the ZIP version of this repository: https://github.com/aaronpdennis/hdm-style.tm2

Unzip the file and you'll have a directory (a folder) called hdm-style.tm2-master. Change the name of this directory to hdm-style.tm2, dropping the -master off the end. This directory is now a Mapbox Studio style that we can browse to and open in the map design software Mapbox Studio.

Before we open the style in Studio, go into the hdm-style.tm2 directory and edit line 60 of the project.yml file where it says source: "mapbox:///mapbox.mapbox-terrain-v2,mapbox.mapbox-streets-v5,aarondennis.humanitarian-data-model-v1". These are all the vector tile sources that this style compiles into one map.

You'll need to change aarondennis.humanitarian-data-model-v1 to your Mapbox mapid for the vector MBTiles you generated and uploaded in the steps above.

Line 60 should now say something like source: "mapbox:///mapbox.mapbox-terrain-v2,mapbox.mapbox-streets-v5,yourusername.your-map-id".

Once you've made these changes, open Mapbox Studio and in the "New Project" window and browse to your hdm-style.tm2 directory. In the settings panel, you'll be able to upload the style to Mapbox, which will then serve your map tiles from the Mapbox API. Here's a good resource for integrating Mapbox Styles into Leaflet web maps: https://www.mapbox.com/mapbox.js/

For Ubuntu 14.04 (Trusty), if you want to use install osmium through npm you first need to install the right dependencies

...
sudo apt-add-repository --yes ppa:chris-lea/node.js
sudo apt-add-repository --yes ppa:ubuntu-toolchain-r/test
sudo apt-get -y update
sudo apt-get -y install git gcc-4.8 g++-4.8 build-essential nodejs
sudo apt-get -y install libboost-dev zlib1g-dev protobuf-compiler
sudo apt-get -y install libprotobuf-dev libexpat1-dev
sudo apt-get -y install libsparsehash-dev
export CC=gcc-4.8
export CXX=g++-4.8
git clone https://github.com/scrosby/OSM-binary.git
cd OSM-binary/src
make && sudo make install
...

hi, if i wish only to create large file dumps of GeoJSON features fot this cuba-latest.osm.pbf, and then use typpecanoe to generate de .mbtiles, is mandatory to use hdm.json, what are the functions of *.json? where are they in the map?