Process and generate Big files with NodeJS

copy-big-file.js

const fs = require('fs');
const path = require('path');

const readStream = fs.createReadStream(path.join(__dirname, 'generated-big-file.csv'));
const writeStream = fs.createWriteStream(path.join(__dirname, 'generated-big-file-copy.csv'));

readStream.pipe(writeStream);

readStream.on('end', () => writeStream.end());

from-stdin-libe-by-line.js

const readline = require('readline');

const { stdin, stdout } = process;

const rl = readline.createInterface({
  input: stdin,
  output: stdout,
  terminal: false,
});

rl.on('line', line => console.log(line));

from-stdin-to-stdout.js

const { stdin, stdout } = process;

stdin.pipe(stdout);

generate-big-file.js

const fs = require('fs');
const path = require('path');

const REGISTERS = 100000000;
const CHUNK_SIZE = 100000;
const stream = fs.createWriteStream(path.join(__dirname, 'generated-big-file.csv'));

let i = 0;

const write = () => {
    if (i >= REGISTERS) {
        stream.end();
        return;
    }

    let chunk = '';

    for (let j = i; j < (i + CHUNK_SIZE); j++) {
        chunk += `${j}, Jesús, Urrutia, [email protected], male, 1985-12-18\n`;
    }
    
    stream.write(chunk, write);

    i += CHUNK_SIZE;
}

write();

infinite.js

let i = 0;
const next = () => process.stdout.write(`${i++}\n`, next);
next();

transform-stream.js

const process = require('process');
const { Transform } = require('stream');

const UpperCaseTransform = new Transform({
    transform(chunk, encoding, callback) {
        this.push(chunk.toString().toUpperCase());
        callback();
    }
});

process.stdin.pipe(UpperCaseTransform).pipe(process.stdout);