Paginating ripple-rest responses with streams

pagination_streams.js

#!/usr/bin/env node

var oboe      = require('oboe');
const address = 'rQE5Z3FgVnRMbVfS6xiVQFgB4J3X162FVD';
const baseUrl = 'http://localhost:5990/v1/accounts/' + address + '/orders?limit=10';
const stream  = process.stdout;

stream.write('[\n');
streamOrders(baseUrl,true)

function done() {
  stream.write('\n]\n');
  process.exit();
}

function streamOrders(url, first) {
  var ledger;
  var marker;

  oboe(url)
  .on('node', {
    'orders.*': function(ord) {
      if(!first) stream.write(',\n');

      first = false;
      stream.write(JSON.stringify(ord,null,'\t'));
    },
    'marker': function(res) {
      marker = res;
    },
    'ledger': function(res) {
      ledger = res;
    }
  })
  .on('done', function() {
    if (!marker) {
      done();
    } else {
      return streamOrders(nextPageUrl(marker, ledger), false);
    }
  })
  .on('fail', function(err) {
    console.log('FAILURE:  ',err);
  });
}

function nextPageUrl(marker, ledger) {
  return baseUrl + '&marker='+marker+'&ledger='+ledger;
}

Using jq to count the number of orders in the fully paginated response:

 » node pagination_streams.js | jq '. | length'
78