Small handy utility script to break large jsonl files to smaller json files.

jsonl-to-multiple-json-files.js

// COUNT=3 INPUT=large_file.jsonl node jsonl-to-multiple-json-files.js
const fs = require('fs');
const readline = require('readline');
const COUNT = parseInt(process.env.COUNT); // Numer of lines to get from the large file
const input = fs.createReadStream(process.env.INPUT.toString()); // input .jsonl file
const rl = readline.createInterface({ input });

let counter = 0;

rl.on('line', (line) => {
  rl.pause();
  if (counter >= COUNT) {
    return rl.close();
  }
  counter++;
  const output = fs.openSync(`./output/file_${counter}.json`, 'w');
  fs.write(output, JSON.stringify(JSON.parse(line), null, 2));
  rl.resume();
}).on('close', () => {
  console.log(`${counter} lines were processed.`);
  process.exit(0);
}).on('error', (error) => {
  console.error(error);
});