Comparison of recursive directory listing methods in JavaScript

recursiveDir.js

/*
 * Comparison of methods to list all files in a directory
 * and subdirectories ("recursive directory walk").
 *
 * gunargessner.com 2015-12-25
 *
 */
var fs = require('fs');
var fsPath = require('path');
var _ = require('highland');

// Synchronous + Imperative Style
var walkImperative = function walkImperative(root) {
  var i, file, dir, results = [], dirs = [root];
  while (dirs.length) {
    var files = fs.readdirSync(dirs[0]);
    dir = dirs[0];
    for (i = 0; i < files.length; i++) {
      file = fsPath.join(dir, files[i]);
      if (fs.lstatSync(file).isDirectory()) dirs.push(file);
      else results.push(file);
    }
    dirs.shift();
  }
  return results;
};

// Synchronous + Functional style
var walkSync = function walkSync(dir) {
  if (fs.lstatSync(dir).isFile()) {
    return [dir];
  } else {
    return fs.readdirSync(dir).map(function (file) {
      file = fsPath.join(dir, file);
      return walkSync(file);
    }).reduce(function (pre, cur) {
      return pre.concat(cur);
    }, []);
  }
};

// Asynchronous + Callbacks
var walkAsync = function walkAsync(dir, cb) {
  fs.lstat(dir, function (err, stat) {
    if (stat.isFile()) {
      cb([dir]);
    } else {
      fs.readdir(dir, function (err, files) {
        var remaining = files.length;
        var out = [];
        files.map(function (file) {
          file = fsPath.join(dir, file);
          walkAsync(file, function (files) {
            out = out.concat(files);
            if (!--remaining) {
              cb(out);
            }
          });
        });
      });
    }
  });
};

// Asynchronous + Promises
function walkPromises(dir) {

  // fs.lstat from cb-style to promise
  const lstat = file =>
    new Promise((resolve, reject) =>
      fs.lstat(dir, (err, stat) => {
        if (err) return reject(err);
        resolve(stat);
      })
    );
  // fs.readdir from cb-style to promise
  const readdir = dir =>
    new Promise((resolve, reject) =>
      fs.readdir(dir, (err, files) => {
        if (err) return reject(err);
        resolve(files);
      })
    );

    return lstat(dir)
      .then(stat => {
        if (stat.isFile()) {
          return [dir];
        } else {
          return readdir(dir)
            .then(files => 
              Promise.all(files.map(file => {
                file = fsPath.join(dir, file);
                return walkPromises(file);
              }))
            );
        }
      }).then(files => {
        return files.reduce(function (pre, cur) {
          return pre.concat(cur);
        }, []);
      });
}

// Asynchornous + Streams Async-style
var walkStreamAsync = function walkStreamAsync(dir) {
  return _(function (push, next) {
    fs.lstat(dir, function (err, stat) {
      if (stat.isFile()) {
        push(err, dir);
        push(null, _.nil);
      } else {
        fs.readdir(dir, function (err, files) {
          next(_(files).map(function (file) {
            file = fsPath.join(dir, file);
            return walkStreamAsync(file);
          }).merge());
        });
      }
    });
  });
};

// Asynchornous + Streams "Stream-style"
var walkStream = function walkStream(dir) {
  return _.wrapCallback(fs.lstat)(dir)
    .map(function (stat) {
      if (stat.isFile()) return _([dir]);
      else {
        return _.wrapCallback(fs.readdir)(dir).sequence()
          .map(function (file) {
            file = fsPath.join(dir, file);
            return walkStream(file);
          }).merge();
      }
    }).merge();
};

// Asynchornous + Streams "Stream-style" with fork
var walkStreamFork = function walkStreamFork(dir) {
  var isFile = function (stat) { return stat.isFile(); };
  var stat = _.wrapCallback(fs.lstat)(dir);

  var _dir = stat.observe().reject(isFile).map(function () {
    return _.wrapCallback(fs.readdir)(dir).sequence()
      .map(function (file) {
        file = fsPath.join(dir, file);
        return walkStreamFork(file);
      }).merge();
  }).merge();

  var _file = stat.filter(isFile).map(function () {
    return dir;
  });

  return _([_file, _dir]).merge();
};


// Example
var dir = '.';

var sync = walkSync(dir);
var imperative = walkImperative(dir);
var async = walkAsync(dir, function (files) {});
var promises = walkPromises(dir).then(function (files) {});
walkStreamAsync(dir).each(function (file) {});
walkStream(dir).each(function (file) {});
walkStreamFork(dir).each(function (file) {});

This is cool! Just one note: you can use sequence() rather than flatMap(_) to flatten a stream of Arrays. It's not mentioned in the docs, but this is the way it's intended to work (to match the behavior of flatten).

Well done! That is a great set of work.

@mlconnor Thank you! :)

@vqvu This makes sense. Updated now. Thank you!

Also, updated to use ES6 Promises instead of relying on Q library.