fatal crash

index.js

var fs = require('fs')
var request = require('request')
var through = require('through2')
var ndjson = require('ndjson')
var once = require('once')
var pump = require('pump')
var concat = require('concat-stream')
var parallel = require('parallel-transform')
var hyperdb = require('hyperdb')
var db = hyperdb('./npm.db', {valueEncoding: 'json'})

update()

function update (err) {
  if (err) {
    log('Error: %s - retrying in 5s', err.message)
    return setTimeout(update, 5000)
  }

  latestSeq(function (err, seq) {
    if (err) throw err

    seq = Math.max(0, seq - 1) // sub 1 incase of errors

    if (seq) log('Continuing fetching npm data from seq: %d', seq)

    var url = 'https://skimdb.npmjs.com/registry/_changes?heartbeat=30000&include_docs=true&feed=continuous' + (seq ? '&since=' + seq : '')

    pump(request(url), ndjson.parse(), save(), update)
  })
}

function latestSeq (cb) {
  db.get('/latest-seq', function (err, val) {
    if (err || !val) return cb(null, 0)
    var seq = val[0].value
    cb(null, seq)
  })
}

function tick (fn, err, val) {
  process.nextTick(function () {
    fn(err, val)
  })
}

function log (fmt) {
  fmt = '[dat-npm] ' + fmt
  console.error.apply(console, arguments)
}

function save () {
  return through.obj(function (data, enc, cb) {
    var doc = data.doc
    if (!doc) return cb()
    if (data.id.match(/^_design\//)) return cb()
    var key = doc._id
    if (doc._deleted) {
      // TODO
      return cb()
    }
    var versions = Object.keys(doc.versions).map(function (v) {
      return v
    })
    db.put('/modules/' + key, versions, function (err) {
      if (err) return cb(err)
      log('wrote /modules/' + key + '=' + versions + ', seq=' + data.seq)
      db.put('/latest-seq', data.seq, cb)
    })
  })
}

output.log

[dat-npm] wrote /modules/solid-ui=0.2.1,0.4.0,0.4.1,0.5.0, seq=2630308
[dat-npm] wrote /modules/react-redux-segments=0.1.0,0.1.1,0.1.2,0.1.3,0.1.4,0.1.5,0.1.6,0.1.7,0.2.0,0.2.1, seq=2630310
[dat-npm] wrote /modules/constant-algebra=0.0.0,0.0.1,0.0.2,0.0.3, seq=2630311
[dat-npm] wrote /modules/cat-graphql=0.0.1,0.0.2,0.0.3,0.0.4,0.0.5,0.0.6,0.0.7,0.0.8,0.0.9,0.0.10,0.0.11,0.0.12,0.0.13,0.0.14,0.1.0,0.1.1,0.1.2,0.1.3,0.2.0, seq=2630313

<--- Last few GCs --->

 1942830 ms: Mark-sweep 1176.7 (1404.2) -> 1176.7 (1406.2) MB, 2104.5 / 0.0 ms [allocation failure] [GC in old space requested].
 1944864 ms: Mark-sweep 1176.7 (1406.2) -> 1176.7 (1406.2) MB, 2034.0 / 0.0 ms [allocation failure] [GC in old space requested].
 1947018 ms: Mark-sweep 1176.7 (1406.2) -> 1186.5 (1404.2) MB, 2154.0 / 0.0 ms [last resort gc].
 1949013 ms: Mark-sweep 1186.5 (1404.2) -> 1196.2 (1404.2) MB, 1993.9 / 0.0 ms [last resort gc].


<--- JS stacktrace --->

==== JS stack trace =========================================

Security context: 0x3868b263fa99 <JS Object>
    1: parse [native json.js:~61] [pc=0x705fd88b781] (this=0x3868b2646091 <a JSON with map 0x1c587d413b91>,C=0x2b2653cfff39 <Very long string[2284820]>,w=0x3868b2604241 <undefined>)
    2: arguments adaptor frame: 1->2
    3: mapper(aka parseRow) [/mnt/bigdisk/dat-npm/node_modules/ndjson/index.js:16] [pc=0x705fd8e6f72] (this=0x2453017cc509 <a DestroyableTransform with map 0x161ed6068541>,row=0x...

FATAL ERROR: CALL_AND_RETRY_LAST Allocation failed - JavaScript heap out of memory
 1: node::Abort() [node]
 2: 0x10d3f9c [node]
 3: v8::Utils::ReportApiFailure(char const*, char const*) [node]
 4: v8::internal::V8::FatalProcessOutOfMemory(char const*, bool) [node]
 5: v8::internal::Factory::NewRawOneByteString(int, v8::internal::PretenureFlag) [node]
 6: v8::internal::Handle<v8::internal::String> v8::internal::JsonParser<false>::ScanJsonString<true>() [node]
 7: v8::internal::JsonParser<false>::ParseJsonObject() [node]
 8: v8::internal::JsonParser<false>::ParseJsonValue() [node]
 9: v8::internal::JsonParser<false>::ParseJsonObject() [node]
10: v8::internal::JsonParser<false>::ParseJsonValue() [node]
11: v8::internal::JsonParser<false>::ParseJsonObject() [node]
12: v8::internal::JsonParser<false>::ParseJsonValue() [node]
13: v8::internal::JsonParser<false>::ParseJsonObject() [node]
14: v8::internal::JsonParser<false>::ParseJsonValue() [node]
15: v8::internal::JsonParser<false>::ParseJsonObject() [node]
16: v8::internal::JsonParser<false>::ParseJsonValue() [node]
17: v8::internal::JsonParser<false>::ParseJson() [node]
18: v8::internal::Runtime_ParseJson(int, v8::internal::Object**, v8::internal::Isolate*) [node]
19: 0x705fd6060c7
Aborted (core dumped)
max@useast:/mnt/bigdisk/dat-npm$