crossref paginated works metadata streaming archiver

index.js

var request = require('request')

var base = 'https://api.crossref.org/works?filter=type:dataset&rows=1000'

doNext()

function doNext (cursor) {
  if (!cursor) cursor = '*'
  var url = base + '&cursor=' + cursor
  console.error('GET', url)
  request(url, {json: true}, function (err, resp, json) {
    if (err || resp.statusCode > 299) throw err || new Error(resp.statusCode + ': ' + JSON.stringify(json))
    var results = json.message.items
    var next = json.message['next-cursor']
    if (json.status !== 'ok') throw new Error(json)
    results.forEach(function (r) {
      console.log(JSON.stringify(r))
    })
    if (!results.length || !next) return console.error('done')
    doNext(encodeURIComponent(next))
  })
}