scrape-memory.js

'use strict';

const SlowBuffer = require('buffer').SlowBuffer;
let buf_arr = [];
let str_arr = [];

// Allocate a lot of buffers
for (let i = 0; i < 1024 * 4 * 4; i++) {
  buf_arr.push(new SlowBuffer(1024 * 256));
}

// Grab strings from all buffers
for (let i = 0; i < buf_arr.length; i++) {
  let strs = buf_arr[i].toString('binary')
                       .split(/[^\x20-\x7f]+/)
                       .filter(v => { if (v.length > 5) return v; });
  str_arr = str_arr.concat(strs);
  buf_arr[i] = undefined;
}

str_arr = getUnique(str_arr).sort();

console.log(str_arr);
console.log(str_arr.length);


function getUnique(arr) {
  let map = {};
  let new_arr = [];
  for (let i = 0; i < arr.length; i++) {
    if (map[arr[i]])
      continue;
    new_arr.push(arr[i]);
    map[arr[i]] = true;
  }
  return new_arr;
}