List all files in a directory in Node.js recursively in a synchronous fashion

walksync.js

// List all files in a directory in Node.js recursively in a synchronous fashion
var walkSync = function(dir, filelist) {
  var fs = fs || require('fs'),
      files = fs.readdirSync(dir);
  filelist = filelist || [];
  files.forEach(function(file) {
    if (fs.statSync(dir + file).isDirectory()) {
      filelist = walkSync(dir + file + '/', filelist);
    }
    else {
      filelist.push(file);
    }
  });
  return filelist;
};

Isn't there a race condition between fs.statSync(...).isDirectory() and the recursion?

Maybe instead:

const fs = require('fs');
const path = require('path');

function walkSync (dir, filelist = []) {
    fs.readdirSync(dir).forEach(file => {
        const dirFile = path.join(dir, file);
        try {
            filelist = walkSync(dirFile, filelist);
        }
        catch (err) {
            if (err.code === 'ENOTDIR' || err.code === 'EBUSY') filelist = [...filelist, dirFile];
            else throw err;
        }
    });
    return filelist;
}

Forked it to make it a generator function so that space complexity goes from O(n) to O(1) (link to fork with full implementation here).

/**
 * List all files in a directory recursively in a synchronous fashion.
 *
 * @param {String} dir
 * @returns {IterableIterator<String>}
 */
function *walkSync(dir) {
    const files = fs.readdirSync(dir);

    for (const file of files) {
        const pathToFile = path.join(dir, file);
        const isDirectory = fs.statSync(pathToFile).isDirectory();
        if (isDirectory) {
            yield *walkSync(pathToFile);
        } else {
            yield pathToFile;
        }
    }
}

Hi, here's a modern version:

const fs = require('fs').promises;
const path = require('path');

const walk = async (dir, filelist = []) => {
  const files = await fs.readdir(dir);

  for (file of files) {
    const filepath = path.join(dir, file);
    const stat = await fs.stat(filepath);

    if (stat.isDirectory()) {
      filelist = await walk(filepath, filelist);
    } else {
      filelist.push(file);
    }
  }

  return filelist;
}

Read as hierarchy tree

    const walkSync = (dir, filelist = []) => {
      const files = fs.readdirSync(dir);
      for (const file of files) {
        const dirFile = path.join(dir, file);
        const dirent = fs.statSync(dirFile);
        if (dirent.isDirectory()) {
          console.log('directory', path.join(dir, file));
          var odir = {
            file: dirFile,
            files: []
          }
          odir.files = walkSync(dirFile, dir.files);
          filelist.push(odir);
        } else {
          filelist.push({
            file: dirFile
          });
        }
      }
      return filelist;
    };

Hi,
this is an excerpt from a project of mine to make a database of all the MP3 I have, using recursive-readdir. It takes around 40 mins for 18641 files. I was wondering how to shorten this time using multiple processes to scan different parts of the file system's portion.

        const promises = [];
        promises.push(readdir(thePath, [function ignoreFiles(file, stats) {
            if (stats.isDirectory()) {
                return false;
            }

            return !isMp3(file);
        }]));

Throwing another iterative (non recursive) one in here. It won't follow symlinks:

const fs = require('fs');
const path = require('path');

module.exports = (dir) => {
  const result = [];

  const files = [dir];
  do {
    const filepath = files.pop();
    const stat = fs.lstatSync(filepath);
    if (stat.isDirectory()) {
      fs
        .readdirSync(filepath)
        .forEach(f => files.push(path.join(filepath, f)));
    } else if (stat.isFile()) {
      result.push(path.relative(dir, filepath));
    }
  } while (files.length !== 0);

  return result;
};

Shameless self promotion. I now use smart-fs

an FP approach

const { join } = require('path');
const { readdirSync, statSync } = require('fs');
const {
  concat, difference, filter, map, reduce,
} = require('lodash/fp');

const getFilePaths = (path, encoding) => {
  const entries = readdirSync(path, { encoding });
  const paths = map(entry => join(path, entry), entries);
  const filePaths = filter(entryPath => statSync(entryPath).isFile(), paths);
  const dirPaths = difference(paths, filePaths);
  const dirFiles = reduce((prev, curr) => concat(prev, getFilePaths(curr)), [], dirPaths);
  return [...filePaths, ...dirFiles];
};

module.exports = getFilePaths;

Async, EJS6, and showing full path.
As few packages as possible.
Tested in Node 10.16.0

const fs = require('fs').promises
const path = require('path')

async function walk(dir, fileList = []) {
  const files = await fs.readdir(dir)
  for (const file of files) {
    const stat = await fs.stat(path.join(dir, file))
    if (stat.isDirectory()) fileList = await walk(path.join(dir, file), fileList)
    else fileList.push(path.join(dir, file))
  }
  return fileList
}

walk('/home/').then((res) => {
  console.log(res)
})

@SamMaxwell approach without using lodash

const fs = require('fs');
const path = require('path');

/** Retrieve file paths from a given folder and its subfolders. */
const getFilePaths = (folderPath) => {
  const entryPaths = fs.readdirSync(folderPath).map(entry => path.join(folderPath, entry));
  const filePaths = entryPaths.filter(entryPath => fs.statSync(entryPath).isFile());
  const dirPaths = entryPaths.filter(entryPath => !filePaths.includes(entryPath));
  const dirFiles = dirPaths.reduce((prev, curr) => prev.concat(getFilePaths(curr)), []);
  return [...filePaths, ...dirFiles];
};

module.exports = getFilePaths;

rxjs 6.5, Typescript 3.5, node 10.6, modifying @quantumsheep's solution above, including both files and directories in a single pass, streaming results into an Observable of a custom type:

import { Stats, promises as fs } from 'fs';
import { Observable } from 'rxjs';

const dirsAndFiles = (dir: string): Observable<FileObject> =>
  new Observable(
    (observer): void => {
      // Create an async recursive function for walking the directory tree:
      const walk = async (dir: string): Promise<void> => {
        const files = await fs.readdir(join(__dirname, dir));
        for (const file of files) {
          const filepath = join(dir, file);
          const stat: Stats = await fs.stat(join(__dirname, filepath));
          if (stat.isFile() || stat.isDirectory()) {
            observer.next({
              isFile: stat.isFile(),
              filename: file,
              size: stat.size,
              path: dir
            });
          }
          if (stat.isDirectory()) {
            await walk(filepath);
          }
        }
      };
      // now call that async function and complete the Observable when it resolves
      walk(dir)
        .then((): void => observer.complete())
        .catch((err): void => observer.error(err));
    }
  );

// later in code:
dirsAndFiles(dir).subscribe(
  (file: FileObject) => doSomething(file),
  (err): void => processError(err)
);

All fine, but how do you create an object than simulates the structure of the actual directory structure?

Like, from this:

Fig 1

LEVEL_1
	LEVEL_2
	|	LEVEL_3_1
	|	|	FILE_3_1_1
	|	|	FILE_3_1_2
	|	LEVEL_3_2
	|	|	FILE_3_2_1
	|	|	FILE_3_2_2
	|	|	LEVEL_4
	|	|	|	FILE_4_1
	|	|	|	FILE_4_2
	|	|	|	... this could go on forever ...
	|	FILE_2_1
	|	FILE_2_2
	FILE_1_1
	FILE_1_2

to this:

Fig 2

{
	LEVEL_2 : {
		LEVEL_3_1 : {
			FILE_3_1_1 : "FILE CONTENT",
			FILE_3_1_2 : "FILE CONTENT"
		},
		LEVEL_3_2 : {
			FILE_3_2_1 : "FILE CONTENT",
			FILE_3_2_2 : "FILE CONTENT"
			LEVEL_4 : {
				FILE_4_1 : "FILE CONTENT",
				FILE_4_2 : "FILE CONTENT"
			}
		},
		FILE_1_1 : "FILE CONTENT",
		FILE_2_1 : "FILE CONTENT"
	}
}

good question, same here... anyone with ES5 old school simple solution?

ouuu hell yea... I love google and I love open source comunity... works just like that:

https://www.npmjs.com/package/directory-tree

const dirTree = require("directory-tree");
const tree = dirTree("/Users/admin/Music");

console.log(tree)

thats all... the rest is up to Angular-tree-control xDDD good luck

*edit - solution #2 https://www.npmjs.com/package/dree NOT tested, just another option

These return the structure alright in some 'weird' way, with all sorts of properties. My own solution to the problem with help from guys over at stackoverflow is more the kind I was looking for: https://stackoverflow.com/a/57626051/8082874

It return a straight object with folders as objects, filenames as keys – and in this case file contents as precompiled Handlebars templates ... like this:

{
	admin: {
		admin_subviews: {
			index: [Function]
		},
		header: {
			[Function: ret] isTop: true,
			_setup: [Function],
			_child: [Function]
		},
		index: {
			[Function: ret] isTop: true,
			_setup: [Function],
			_child: [Function]
		},
		layout: {
			[Function: ret] isTop: true,
			_setup: [Function],
			_child: [Function]
		}
	},
	index:{
		[Function: ret] isTop: true,
		_setup: [Function],
		_child: [Function]
	},
	layout:	{
		[Function: ret] isTop: true,
		_setup: [Function],
		_child: [Function]
	},
	login: {
		index: {
			[Function: ret] isTop: true,
			_setup: [Function],
			_child: [Function]
		},
		layout: {
			[Function: ret] isTop: true,
			_setup: [Function],
			_child: [Function]
		}
	}
}

The root level is the object itself and you can manipulate the file content in any way you want and inject it as the value.

I have no idea what you mean but for me it just rendered JSON on server of all files/folders in tree -> send that tree to front-end-> and https://github.com/wix/angular-tree-control just directly render all of that at a glance 👍 but you look like that you need more than that...

Probably yes. And whenever something contains 'angular' I'm usually out anyway, so ; )

yea xD I'm locked in the 2015 year development and ES5 but I'm so grateful for that but depends on everybody's needs ;) good luck anyway

Here's an ES6 refactor. One that applies RegExp filtering:

const fs = require('fs');
const path = require('path');

function findInDir (dir, filter, fileList = []) {
  const files = fs.readdirSync(dir);

  files.forEach((file) => {
    const filePath = path.join(dir, file);
    const fileStat = fs.lstatSync(filePath);

    if (fileStat.isDirectory()) {
      findInDir(filePath, filter, fileList);
    } else if (filter.test(filePath)) {
      fileList.push(filePath);
    }
  });

  return fileList;
}

// Usage
findInDir('./public/audio/', /\.mp3$/);

@meowsus Your solution works great, thanks.

Just another implementation

const path = require('path');
const fs = require('fs');

/* Prepend the given path segment */
const prependPathSegment = pathSegment => location => path.join(pathSegment, location);

/* fs.readdir but with relative paths */
const readdirPreserveRelativePath = location => fs.readdirSync(location).map(prependPathSegment(location));

/* Recursive fs.readdir but with relative paths */
const readdirRecursive = location => readdirPreserveRelativePath(location)
    .reduce((result, currentValue) => fs.statSync(currentValue).isDirectory()
        ? result.concat(readdirRecursive(currentValue))
        : result.concat(currentValue), []);

I think the most elegant way to list all files and/or directories recursively is using globs. You can use globby.

const globby = require('globby');

const listAllFilesAndDirs = dir => globby(`${dir}/**/*`);

(async () => {
  const result = await listAllFilesAndDirs(process.cwd());
  console.log(result);
})();

@gemyero Great solution, thanks!

mmh Carefull with globby ;)

Here is my way (with typescript):

import fs from "fs/promises";
import path from "path";

export default async function walk(directory: string) {
  let fileList: string[] = [];

  const files = await fs.readdir(directory);
  for (const file of files) {
    const p = path.join(directory, file);
    if ((await fs.stat(p)).isDirectory()) {
      fileList = [...fileList, ...(await walk(p))];
    } else {
      fileList.push(p);
    }
  }

  return fileList;
}

Alternative solution in Just two lines, Just for fun and learning

const path = ''; // 👈 path to your location
const list = require("child_process").execSync(`cd ${path} && ls -R`).toString().split(`\n`);

⚠️ IMPORTANT
Be careful, this solution use a bash script and it works fine in MACOS, for windows or linux could be different

What I use is based on @vidul-nikolaev-petrov version, with a callback argument to do something with files:

const walkSync = (dir, callback) =>  fs.lstatSync(dir).isDirectory()
        ? fs.readdirSync(dir).map(f => walkSync(path.join(dir, f), callback))
        : callback(dir);

// Note: call to flat at the end to have one array with every paths
const svgs = walkSync(path, fileToBase64).flat()

Alternative solution in Just two lines, Just for fun and learning

const path = ''; // 👈 path to your location
const list = require("child_process").execSync(`cd ${path} && ls -R`).toString().split(`\n`);

⚠️ IMPORTANT
Be careful, this solution use a bash script and it works fine in MACOS, for windows or linux could be different

Please, never do that

@FerreiraRaphael kudos

Hi, here's a modern version:

const fs = require('fs').promises;
const path = require('path');

const walk = async (dir, filelist = []) => {
  const files = await fs.readdir(dir);

  for (file of files) {
    const filepath = path.join(dir, file);
    const stat = await fs.stat(filepath);

    if (stat.isDirectory()) {
      filelist = await walk(filepath, filelist);
    } else {
      filelist.push(file);
    }
  }

  return filelist;
}

Here's what I used (needed import)

import * as fs from 'fs/promises';
import path from 'path';

export async function directory_read(dir, filelist = []) {
  const files = await fs.readdir(dir);

  for (let file of files) {
    const filepath = path.join(dir, file);
    const stat = await fs.stat(filepath);

    if (stat.isDirectory()) {
      filelist = await directory_read(filepath, filelist);
    } else {
      filelist.push(file);
    }
  }

  return filelist;
}

Node v20.1+

const filelist = await fs.readdir(dir, { recursive: true });

TypeScript version with filters:

import {readdirSync, statSync} from "node:fs";
import * as path from "node:path";

/**
 * Options for scanning a directory.
 *
 * @template PathType - The type of the relative directory path.
 * @interface DirectoryScanOptions
 * @property {string | PathType} [baseDir=""] - The base directory for relative paths.
 * @property {string[]} [fileExtensions=[]] - An array of file extensions to filter.
 * @property {"all" | "dir" | "file"} [collectionType="all"] - Specifies what to collect:
 *   - "all": collect both files and directories
 *   - "dir": collect only directories
 *   - "file": collect only files
 */
interface DirectoryScanOptions<PathType extends string = ""> {
    /**
     * Optional relative directory path
     *
     * The base directory for relative paths.
     *
     * @default ""
     */
    baseDir?: string | PathType;

    /**
     * Optional array of file extensions
     *
     * An array of file extensions to filter.
     * 
     * @default []
     */
    fileExtensions?: string[];

    /**
     * Optional collection type
     *
     * Specifies what to collect:
     *  - "all": collect both files and directories
     *  - "dir": collect only directories
     *  - "file": collect only files
     *
     *  @default "all"
     */
    collectionType?: "all" | "dir" | "file";
}

interface ScanDirectory {

    /**
     * Scans a directory and collects paths based on specified options.
     *
     * @template PathType - The type of the directory path.
     * @param {PathType} targetDir - The path of the directory to scan.
     * @returns {string[]} - An array of collected paths relative to the specified directory.
     *
     * @example
     * // Example usage of scanDirectory function
     * const collectedPaths = scanDirectory('/path/to/directory', {
     *     baseDir: '/path/to',
     *     fileExtensions: ['.js', '.ts'],
     *     collectionType: 'file'
     * });
     * console.log(collectedPaths); // Outputs an array of relative file paths
     */
    <PathType extends string>(targetDir: PathType): string[];

    /**
     * Scans a directory and collects paths based on specified options.
     *
     * @template PathType - The type of the directory path.
     * @param {PathType} targetDir - The path of the directory to scan.
     * @param {DirectoryScanOptions<PathType>} [options] - Options for scanning the directory.
     * @returns {string[]} - An array of collected paths relative to the specified directory.
     *
     * @example
     * // Example usage of scanDirectory function
     * const collectedPaths = scanDirectory('/path/to/directory', {
     *     baseDir: '/path/to',
     *     fileExtensions: ['.js', '.ts'],
     *     collectionType: 'file'
     * });
     * console.log(collectedPaths); // Outputs an array of relative file paths
     */
    <PathType extends string>(targetDir: PathType, options: DirectoryScanOptions<PathType>): string[];
}

/**
 * Scans a directory and collects paths based on specified options.
 *
 * @template PathType - The type of the directory path.
 * @param {PathType} targetDir - The path of the directory to scan.
 * @param {DirectoryScanOptions<PathType>} [options] - Options for scanning the directory.
 * @returns {string[]} - An array of collected paths relative to the specified directory.
 *
 * @example
 * // Example usage of scanDirectory function
 * const collectedPaths = scanDirectory('/path/to/directory', {
 *     baseDir: '/path/to',
 *     fileExtensions: ['.js', '.ts'],
 *     collectionType: 'file'
 * });
 * console.log(collectedPaths); // Outputs an array of relative file paths
 */
export const scanDirectory: ScanDirectory = <PathType extends string>(
    targetDir: PathType, // The directory path to scan
    options?: DirectoryScanOptions<PathType> // Options for scanning a directory.
): string[] => {

    const {
        baseDir = "", // The base directory for relative paths
        fileExtensions = [], // Default to an empty array if no extensions are provided
        collectionType = "all" // Default to collecting all types
    } = options || {};

    // Type alias for the collection options
    type CollectionArray = Array<typeof collectionType>;

    // Array to hold the scanned paths
    const collectedPaths: string[] = [];
    // Read directory contents
    const directoryContents = readdirSync(targetDir, {withFileTypes: true, recursive: true});
    // Check if any extensions are provided
    const hasExtensions = !!fileExtensions?.length;

    // Iterate over each file/directory in the scanned directory
    for (const directoryEntry of directoryContents) {
        // Construct the full file path
        const fullPath = path.join(directoryEntry.parentPath, directoryEntry.name);
        // Get the file statistics
        const fileStats = statSync(fullPath);
        // Get the relative path
        const relativePath = path.relative(baseDir, fullPath);

        // Check if we should collect files
        if ((["all", "file"] as CollectionArray).includes(collectionType) && fileStats.isFile()) {
            // If extensions are specified, check if the file matches
            if (hasExtensions && !fileExtensions.includes(path.extname(fullPath))) {
                continue; // Skip this file if it doesn't match the extensions
            }
            // Add the relative file path to the collected array
            collectedPaths.push(relativePath);
        }
        // Check if we should collect directories
        else if ((["all", "dir"] as CollectionArray).includes(collectionType) && fileStats.isDirectory()) {
            // Add the relative directory path to the collected array
            collectedPaths.push(relativePath);
        }
    }

    // Return the collected paths
    return collectedPaths;
};

Vanilla JS version if you don't use TypeScript:

import {readdirSync, statSync} from "node:fs";
import * as path from "node:path";

/**
 * Scans a directory and collects paths based on specified options.
 *
 * @template PathType - The type of the directory path.
 * @param {PathType} targetDir - The path of the directory to scan.
 * @param {DirectoryScanOptions<PathType>} [options] - Options for scanning the directory.
 * @returns {string[]} - An array of collected paths relative to the specified directory.
 *
 * @example
 * // Example usage of scanDirectory function
 * const collectedPaths = scanDirectory('/path/to/directory', {
 *     baseDir: '/path/to',
 *     fileExtensions: ['.js', '.ts'],
 *     collectionType: 'file'
 * });
 * console.log(collectedPaths); // Outputs an array of relative file paths
 */
export const scanDirectory = (
    targetDir, // The directory path to scan
    options // Options for scanning a directory.
) => {
    const {
        baseDir = "", // The base directory for relative paths
        fileExtensions = [], // Default to an empty array if no extensions are provided
        collectionType = "all" // Default to collecting all types
    } = options || {};

    // Array to hold the scanned paths
    /** @type {string[]} */
    const collectedPaths = [];
    // Read directory contents
    /** @type {Dirent[]} */
    const directoryContents = readdirSync(targetDir, {withFileTypes: true, recursive: true});
    // Check if any extensions are provided
    /** @type {boolean} */
    const hasExtensions = !!fileExtensions?.length;

    // Iterate over each file/directory in the scanned directory
    for (const directoryEntry of directoryContents) {
        // Construct the full file path
        /** @type {string} */
        const fullPath = path.join(directoryEntry.parentPath, directoryEntry.name);
        // Get the file statistics
        /** @type {Stats} */
        const fileStats = statSync(fullPath);
        // Get the relative path
        /** @type {string} */
        const relativePath = path.relative(baseDir, fullPath);
        // Check if we should collect files
        if (["all", "file"].includes(collectionType) && fileStats.isFile()) {
            // If extensions are specified, check if the file matches
            if (hasExtensions && !fileExtensions.includes(path.extname(fullPath))) {
                continue; // Skip this file if it doesn't match the extensions
            }
            // Add the relative file path to the collected array
            collectedPaths.push(relativePath);
        }
        // Check if we should collect directories
        else if (["all", "dir"].includes(collectionType) && fileStats.isDirectory()) {
            // Add the relative directory path to the collected array
            collectedPaths.push(relativePath);
        }
    }
    // Return the collected paths
    return collectedPaths;
};

// JSODC types

/**
 * Options for scanning a directory.
 *
 * @template PathType - The type of the relative directory path.
 * @interface DirectoryScanOptions
 * @property {string | PathType} [baseDir=""] - The base directory for relative paths.
 * @property {string[]} [fileExtensions=[]] - An array of file extensions to filter.
 * @property {"all" | "dir" | "file"} [collectionType="all"] - Specifies what to collect:
 *   - "all": collect both files and directories
 *   - "dir": collect only directories
 *   - "file": collect only files
 */