Experiment with different line-iteration methods in node.js

generate-random-data.sh

#!/bin/bash
#
# Generate ~135 MB of random data for iterating over with node.
# A file of this size is large enough where the OS _for sure_ won't try
# reading it all in all at once; it'll iterate over it in chunks.
#
# Adapted from https://superuser.com/a/470957/103045
openssl rand -out sample.txt -base64 $(( 2**27 * 3/4 ))

iterate-by-line-without-readline.js

var os = require('os');
var fs  = require('fs');
var readline = require('readline');

let logfile = 'sample.txt'

/*  This code's structure is adapted from setupInvestLogReaderHandler in the workbench
  * from PR https://github.com/natcap/invest-workbench/pull/167,
  * revision 2611b1e7fc3eafd92e4833c6e5c4785637a15866i
  *
  * */

// this will close on its own (by default) when the whole file is consumed.
const fileStream = fs.createReadStream(logfile);
fileStream.on('error', (err) => {
  console.log(err);
  console.log(`${logfile} is missing or unreadable`);
});

fileStream.on('data', (chunk) => {
  var array = chunk.toString().split("\n");
  for (i in array){
    console.log(`${array[i]}`);
    console.log(`NEWLINE`);
  };
});

iterate-with-readline.js

var os = require('os');
var fs  = require('fs');
var readline = require('readline');

let logfile = 'sample.txt'

/*  This code's structure is adapted from setupInvestLogReaderHandler in the workbench
  * from PR https://github.com/natcap/invest-workbench/pull/167,
  * revision 2611b1e7fc3eafd92e4833c6e5c4785637a15866i
  *
  * */

const fileStream = fs.createReadStream(logfile);
fileStream.on('error', (err) => {
  console.log(err);
  console.log(`${logfile} is missing or unreadable`);
});

const rl = readline.createInterface({
  input: fileStream,
  crlfDelay: Infinity,
});

rl.on('line', (line) => {
  console.log(`${line}`);
  console.log(`NEWLINE`);
});

iterate-without-readline.js

var os = require('os');
var fs  = require('fs');
var readline = require('readline');

let logfile = 'sample.txt'

/*  This code's structure is adapted from setupInvestLogReaderHandler in the workbench
  * from PR https://github.com/natcap/invest-workbench/pull/167,
  * revision 2611b1e7fc3eafd92e4833c6e5c4785637a15866i
  *
  * */

// this will close on its own (by default) when the whole file is consumed.
const fileStream = fs.createReadStream(logfile);
fileStream.on('error', (err) => {
  console.log(err);
  console.log(`${logfile} is missing or unreadable`);
});

fileStream.on('data', (chunk) => {
  console.log(`${chunk}`);
  console.log(`NEWLINE`);
});

Makefile

.PHONY: all clean

WITHREADLINE := withreadline.txt
WITHOUTREADLINE := withoutreadline.txt
WITHOUTREADLINEBYLINE := withoutreadlinebyline.txt
SAMPLE := sample.txt

all: $(WITHREADLINE) $(WITHOUTREADLINE) $(WITHOUTREADLINEBYLINE)

clean:
	rm -f $(SAMPLE) $(WITHREADLINE) $(WITHOUTREADLINE) $(WITHOUTREADLINEBYLINE)

$(SAMPLE):
	bash generate-random-data.sh

$(WITHREADLINE): $(SAMPLE)
	time node iterate-with-readline.js > $(WITHREADLINE)

$(WITHOUTREADLINE): $(SAMPLE)
	time node iterate-without-readline.js > $(WITHOUTREADLINE)

$(WITHOUTREADLINEBYLINE): $(SAMPLE)
	time node iterate-by-line-without-readline.js > $(WITHOUTREADLINEBYLINE)