Benchmark to show performance difference between fs sync and async calls

fs-scan-n-read-bench.js

'use strict';

const MAX_DEPTH = 4;

const fs = require('fs');
const print = process._rawDebug;
const sep = require('path').sep;
const tmpdir = require('os').tmpdir() + sep + 'data-file-bench-' + genName();
const file_list = newDir();
var current_dir = file_list;
var current_path = [ tmpdir ];
var total_size = 0;
var total_files = 0;
var total_folders = 0;

// Making large Buffer so some files will be multi-sector. Also fill buf with
// random data.
const buf = Buffer.alloc(0x2000, Math.random().toString(36));

process.on('SIGINT', process.exit);
process.on('exit', () => {
  cleanupFiles(file_list, [tmpdir]);
});
process.on('uncaughtException', (e) => {
  if (!process._exiting)
    cleanupFiles(file_list, [tmpdir]);
  throw e;
});


{
  const t = process.hrtime();
  genFiles();
  print('total_files:  ', total_files);
  print('total_folders:', total_folders);
  print('total_size:   ', total_size);
  printTime('genFiles:', process.hrtime(t), 1);
}


// Do this so the system caches all the files. So perf test is level.
readAllFilesSync(tmpdir);


{
  const t = process.hrtime();
  readAllFilesSync(tmpdir);
  printTime('readAllFilesSync:', process.hrtime(t), 1);
}


{
  const t = process.hrtime();
  readAllFilesAsync(
      tmpdir, () => printTime('readAllFilesAsync:', process.hrtime(t), 1));
}


function readAllFilesSync(dir) {
  const contents = fs.readdirSync(dir);
  for (const val of contents) {
    const new_path = dir + sep + val;
    const file_stats = fs.lstatSync(new_path);
    if (file_stats.isDirectory())
      readAllFilesSync(new_path);
    else if (file_stats.isFile())
      fs.readFileSync(new_path);
    else
      throw new Error('wtf are you up to?');
  }
}


function readAllFilesAsync(dir, cb) {
  var remaining = null;
  if (typeof cb !== 'function') cb = () => {};
  fs.readdir(dir, (err, contents) => {
    if (err) throw err;
    remaining = contents.length;
    for (const val of contents) {
      const new_path = dir + sep + val;
      fs.lstat(new_path, (err, file_stats) => {
        if (err) throw err;
        if (file_stats.isDirectory()) {
          readAllFilesAsync(new_path, () => {
            if (--remaining <= 0) cb();
          });
        } else if (file_stats.isFile()) {
          fs.readFile(new_path, (err, data) => {
            if (err) throw err;
            if (--remaining <= 0) cb();
          });
        } else {
          throw new Error('seriously?');
        }
      });
    }
  });
}


// Generate a lot of files of different sizes.
function genFiles() {
  // First create the dir that all the others will go in.
  fs.mkdirSync(tmpdir);
  print(`tmp dir for bench is ${tmpdir}`);
  createFilesNDirs();
}

function createFilesNDirs() {
  // First create 16 files.
  for (let i = 1; i <= 16; i++) {
    let file_name = null;
    while (file_name === null) {
      try {
        const tmp_name = genName();
        fs.writeFileSync(ffPath(tmp_name), buf.slice(i * 0x200));
        file_name = tmp_name;
        total_files++;
        total_size += i * 0x200;
      } catch(e) { if (e.code !== 'EEXIST') throw e }
    }
    current_dir.files.push(file_name);
  }

  // All done if max depth has been reached.
  if (current_path.length > MAX_DEPTH) {
    current_path.pop();
    current_dir = current_dir.parent;
    return;
  }

  // Generate 4 folders and populate them.
  for (let i = 0; i < 4; i++) {
    let folder_name = null;
    while (folder_name === null) {
      try {
        const tmp_name = genName();
        fs.mkdirSync(ffPath(tmp_name));
        folder_name = tmp_name;
        total_folders++;
      } catch(e) { if (e.code !== 'EEXIST') throw e }
    }
    current_dir = current_dir.folders[folder_name] = newDir();
    current_path.push(folder_name);
    createFilesNDirs();
  }
  current_path.pop();
  current_dir = current_dir.parent;
}

function cleanupFiles(obj, fpath) {
  for (let dir in obj.folders) {
    fpath.push(dir);
    cleanupFiles(obj.folders[dir], fpath);
  }
  for (const file_name of obj.files) {
    const file_path = ffPath(fpath.join(sep) + sep + file_name);
    try {
      fs.unlinkSync(file_path);
    } catch (e) { if (e.code !== 'ENOENT') throw e }
  }
  fs.rmdirSync(fpath.join(sep));
  fpath.pop();
}


function genName() { return Math.random().toString(36).substr(-8) }
function newDir() { return { files: [], folders: {}, parent: current_dir } }
function ffPath(name) { return current_path.join(sep) + sep + name }


function printTime(msg, t, iter) {
  const op_txt = (iter === 1) ? '' : '/op';
  var ops = (t[0] + t[1] / 1e9) / iter;

  if (ops >= 10) {
    print(msg, ops.toFixed(2), 'sec' + op_txt);
  } else if ((ops *= 1e3) >= 10) {
    print(msg, ops.toFixed(2), 'ms' + op_txt);
  } else if ((ops *= 1e3) >= 10) {
    print(msg, ops.toFixed(2), 'us' + op_txt);
  } else {
    print(msg, (ops * 1e3).toFixed(2), 'ns' + op_txt);
  }
}