How to write two million rows of csv data using faker.js and node cluster

write-two-millio-rows-of-csv-data-with-node-cluster.js

const faker = require('faker')
// const N = 30
const N = 2000000
const fs = require('fs')
const record = () => {
  // faker.fake(
  //   '{{name.lastName}},{{name.lastName}},{{address.city}},{{address.county}},{{address.zipCode}},{{hacker.adjective}}\n'
  return [
    faker.name.firstName(),
    faker.name.lastName(),
    faker.address.city(),
    faker.address.state(),
    faker.address.zipCode(),
    faker.hacker.adjective()
  ]
}
const cluster = require('cluster')
const numCPUs = require('os').cpus().length

let workers = []
let written = 0
if (cluster.isMaster) {
  masterProcess()
} else {
  childProcess()
}

function masterProcess () {
  console.log(`Master ${process.pid} is running`)

  for (let i = 0; i < numCPUs; i++) {
    console.log(`Forking process number ${i}...`)
    const worker = cluster.fork()
    worker.on('message', msg => {
      if (msg === 'done') {
        written += 1
      }
      if (written % 100000 === 0) {
        console.log(`written=${written}`)
      }
      if (written >= N) {
        console.log('all done')
        process.exit()
      }
    })
    workers.push(worker)
  }
  for (let i = 0; i < N; i++) {
    const workerId = i % numCPUs
    const worker = workers[workerId]
    worker.send(i)
  }
}

function childProcess () {
  const writer = fs.createWriteStream('out.csv', { flags: 'a' })
  console.log(`Worker ${process.pid} started`)
  process.on('message', message => {
    const msg = record().join(',') + '\n'
    writer.write(msg, () => {
      // signal writing is done for each entry
      process.send('done')
    })
  })
}

Instruction:

1. create a new folder, download this script and save in it
2. run npm i faker inside that folder
3. run time node write-two-millio-rows-of-csv-data-with-node-cluster.js