extract-geojson-relations.js

var osmium = require('osmium');
var _ = require("lodash")

var count = 0;
var restrictions=0;

var relHandler  = new osmium.Handler();
var geomHandler = new osmium.Handler();

var nodes = []
var nodeLocations = {}
var ways  = []
var wayLocations = {}
var relations = {} //Will just store this in memory after the first pass

relHandler.on('relation', function(relation) {
  count++;
  if (count%100==0){process.stderr.write("\r"+count)}
    if (relation.tags("type")==='restriction'){
      restrictions++;

      try{

        var members = relation.members();

        if (members.length ){

          var via = members.filter((m)=>{return m.role=='via'})[0]
          var from = members.filter((m)=>{return m.role=='from'})[0]
          var to = members.filter((m)=>{return m.role=='to'})[0]

          if(via){
            if(via.type=='n'){
              nodes.push(via.ref)
              var ref = {'type':'n','ref':via.ref}
            }else if(via.type=='w'){
              ways.push(via.ref)
              var ref = {'type':'w','ref':via.ref}
            }
          }else if(from){
            if(from.type=='n'){
              nodes.push(from.ref)
              var ref = {'type':'n','ref':from.ref}
            }else if(from.type=='w'){
              ways.push(from.ref)
              var ref = {'type':'w','ref':from.ref}
            }
          }else if(to){
            if(to.type=='n'){
              nodes.push(to.ref)
              var ref = {'type':'n','ref':to.ref}
            }else if(to.type=='w'){
              ways.push(to.ref)
              var ref = {'type':'w','ref':to.ref}
            }
          }else{
            if(members[0].type=='n'){
              nodes.push(members[0].ref)
              var ref = {'type':'n','ref':members[0].ref}
            }else if(members[0].type=='w'){
              ways.push(members[0].ref)
              var ref = {'type':'w','ref':members[0].ref}
            }
          }

          relations[relation.id] = {
            i: relation.id,
            u: relation.uid,
            h: relation.user,
            t: relation.timestamp_seconds_since_epoch,
            c: relation.changeset,
            v: relation.version,
            tags: relation.tags(),
            ref: ref
          }
        }
      }catch(e){
        console.warn(e)
      }
    }
});

var nodeCount = 0;
geomHandler.on('node',function(node){
  ++nodeCount;
  if (nodeCount%1000000==0){process.stderr.write("\rread "+nodeCount/1000000+"M nodes, remaining geometries: "+nodes.length+"                 ")}

  if (node.id > curNode){
    while (curNode < node.id){
      console.warn("Node" + curNode + " not in file")
      curNode = nodes.pop()
    }
  }

  if (node.id==curNode){
    nodeLocations[node.id] = node.coordinates

    //last step
    curNode = nodes.pop();

  }
})

var wayCount = 0;
geomHandler.on('way',function(way){
  ++wayCount;
  if (wayCount%1000==0){process.stderr.write("\r"+wayCount/1000+"K ways")}

  if (way.id > curWay){
    while (curWay < node.id){
      console.warn("Way" + curWay + " not in file")
      curWay = nodes.pop()
    }
  }

  if (way.id==curWay){
    var repNode = way.node_refs(0)
    nodes.push( repNode ) //push that node, mark it.

    wayLocations[curWay] = repNode

    //last step
    curWay = ways.pop()
  }
})

geomHandler.on('done', function() {

  //Now matching geometries and creating geojson
  Object.keys(relations).forEach(function(relId){

    try{
      var thisRel = relations[relId]
      var geom;
      if (thisRel.ref.type=='n'){
        geom = {'type':"Point",'coordinates':[nodeLocations[thisRel.ref.ref].lon,nodeLocations[thisRel.ref.ref].lat]}
      }

      if (thisRel.ref.type=='w'){
        repNode = wayLocations[thisRel.ref.ref]
        geom = {'type':"Point",'coordinates':[nodeLocations[repNode].lon, nodeLocations[repNode].lat]}
      }

      var props = thisRel.tags;
      delete props.ref
      props['@id']        = thisRel.i
      props['@user']      = thisRel.h
      props['@uid']       = thisRel.u
      props['@timestamp'] = thisRel.t
      props['@changeset'] = thisRel.c
      props['@version']   = thisRel.v
      props['@tr']        = true

      console.log(JSON.stringify(
        {'type':"Feature",
        "geometry":geom,
        "properties":props}))
    }catch(e){
      console.warn(e)
    }
  })

  console.warn("\nFinished, nodes left in nodes array: " + nodes.length)
})


relHandler.on('done', function() {

  //Sort them... and hope that the pbf file is ordered the way we hope it is.
  console.warn(`\rProcessed ${count} relations, found ${restrictions} restrictions`)
  console.warn(`Nodes Involved: ${nodes.length}`)
  console.warn(`Ways Involved:  ${ways.length}`)
  console.warn(`Missing points: ${restrictions - nodes.length - ways.length}\n`);

  nodes = _.uniq(_.sortBy(nodes,function(x){return -Number(x)}));
  ways  = _.uniq(_.sortBy(ways,function(x){return -Number(x)}));

  console.warn(`Nodes Needed: ${nodes.length}`)
  console.warn(`Ways Needed:  ${ways.length}`)

  curNode = nodes.pop();
  curWay = ways.pop();
});

/*
  Runtime
*/

if (!process.argv[2]) {
    console.error('Usage: \n');
    console.error('\tnode extract-geojson-relations.js [OSM FILE] > [GEOJSONSEQ FILE]');
    console.error("\n\n")
    process.exit(1);
}

var input_filename = process.argv[2]


console.warn("----------------------------------------------------------------------")
console.warn("Stage 1: Getting ways / nodes lists from restriction relations\n")

var reader = new osmium.Reader(input_filename, {node: false, way: false, relation:true});
osmium.apply(reader, relHandler);
relHandler.end();
reader.close();

console.warn("----------------------------------------------------------------------")
console.warn("Stage 2: Getting representative NODES for " + ways.length + " ways")
reader = new osmium.Reader(input_filename, {node: false, way: true, relation:false});
osmium.apply(reader, geomHandler);
reader.close();

nodes = _.uniq(_.sortBy(nodes,function(x){return -Number(x)}));
console.warn(`\nNodes Now Needed: ${nodes.length}`)

console.warn("---------------------------------------------------------------------")
console.warn("Stage 3: Getting Node geometries")
reader = new osmium.Reader(input_filename, {node: true, way: false, relation:false});

osmium.apply(reader, geomHandler);
geomHandler.end();
reader.close();

Putting Turn Restrictions in OSM-QA-Tiles?

This is a prototype of using node-osmium to process the latest planet file and extract all turn restrictions.

Turn restrictions are then represented by a single point; this is useful for seeing where and when turn restrictions are edited on the map...

Briefly, it does the following (without using a location handler to improve performance because it's so sparse):

1. Identifies all of the turn restrictions in an OSM file and represent it as a single point: preferably the point of the via node if it is a node, otherwise pull the first node out of one of the ways from another member to represent the point.

2. Create a geojson feature for each point with the properties matching the osm-qa-tile schema

3. Pipe it out to a geojsonseq file or tippecanoe!

   $ tippecanoe -l osm -Z12 -z12 -d20 -pf -pk -ps --no-duplication --no-tile-stats -b0 -pd -Pf -o turn-restrictions.mbtiles turn-restrictions.geojsonseq
   

4. Should be able to use tile-join to add these features to osm-qa-tiles.... and now osm-qa-tiles can have turn restrictions 👍

   $ tile-join -pg -pk -f -o osm-qa-with-tr.mbtiles latest.planet.mbtiles turn-restrictions.mbtiles
   

Implementation

Takes 1-2 hours to run on full planet file, just over 1M turn restrictions. Requires parsing the planet file 3 times, but each type of OSM element only once, and doesn't use any location_handlers, so it's pretty fast.

Ultimately, it could be faster in C++, porting this node proof-of-concept shouldn't be too complicated.

Thoughts

Wouldn't this be much faster to lookup in a database? Yes, but this is meant to be a stand-alone utility with no uptime dependencies.

Next Steps

• Can certainly expand to other types of relations
• Could use different geometries depending on type of relation
• Would be great to incorporate into something like osmium-export eventually

Here is a suggestion: Use Osmium first to get all the turn restrictions filtered from the planet file: osmium tags-filter planet.osm.pbf r/type=restriction -o restrictions.osm.pbf. This runs in less than 10 minutes on my test system. Then run your script on the result. Should be much faster this way.

Thanks @joto - this is a great suggestion, will test on my next run.... thinking about how the r/type=restriction filter likely works (and requires parsing the file 3x?), I imagine these are very similar processes...

Could probably be refactored to simply run the osmium tags-filter to get all of the necessary objects and then do a single run through the output converting to GeoJSON on the fly (just hold all 1M node / way locations in memory)... will test down the road.

Thinking more down the road, what are the big red flags that you see with such geometric representations of these objects, and do you see value in defining / standardizing such objects as GeoJSON for an eventual --include-restrictions tag in something like osmium-export?