graceful-vfs-src-with-gs-and-pipeline.js

const fs = require('fs')
const gs = require('glob-stream')
const File = require('vinyl')
const { Transform } = require('stream')
const map = (transform) => new Transform({ objectMode: true, transform })
const ospath = require('path')
//const isUTF8 = require('is-utf8')
const { pipeline } = require('stream')

function smartStat (path_, callback) {
  fs.lstat(path_, (lstatErr, lstat) => {
    if (lstatErr) {
      callback(lstatErr) // NOTE extremely rare
    } else if (lstat.isSymbolicLink()) {
      fs.stat(path_, (statErr, stat) => {
        //statErr ? callback(Object.assign(statErr, { link: true })) : callback(null, stat)
        if (statErr) {
          fs.readlink(path_, (_, link) => {
            callback(Object.assign(statErr, { link }))
          })
        } else {
          callback(null, stat)
        }
      })
    } else {
      callback(null, lstat)
    }
  })
}

;(async () => {
  await new Promise((resolve, reject) => {
    console.time('measure')
    const accum = []
    const cache = {}
    const cwd = ospath.resolve('../docs')
    pipeline(
      gs('**/*', {
        cwd,
        //cwd: ospath.resolve('more-docs'),
        follow: true,
        nomount: true,
        nosort: true,
        uniqueBy: ((entries) => entries),
        strict: false, // silently ignore files that cannot be read
        cache, // we could reuse cache, but it's very low level and not likely to provide all the info we need
        //nodir: true, // don't return directories, though it seems to come with a slight cost; but why?
      }),
      map(({ path: abspath }, _, next) => {
        //to skip directories eagerly, map cache and enable this line
        //if (Array.isArray(cache[abspath])) return next()
        smartStat(abspath, (statErr, stat) => {
          if (statErr) {
            console.warn('skipping file: ' + abspath + (statErr.link ? ` (${statErr.code === 'ELOOP' ? 'symlink cycle' : 'broken symlink'}: ${statErr.link})` : ''))
            return next()
          }
          if (stat.isDirectory()) return next()
          fs.readFile(abspath, (readErr, contents) => {
            if (readErr) {
              console.warn('skipping file: ' + abspath)
              return next()
            }
            //if (contents[0] === 0xEF && contents[1] === 0xBB && contents[2] === 0xBF && isUTF8(contents)) {
            //  contents = contents.slice(3)
            //}
            accum.push(new File({ base: cwd, cwd, path: abspath, contents, stat }))
            next()
          })
        })
      }),
      (err) => err ? reject(err) : resolve(accum)
    )
  })
    .then((files) => {
      console.timeEnd('measure')
      console.log(files.length)
      console.log(files[1].relative)
      //files.forEach((file) => console.log(file.relative))
    })
    .catch((err) => {
      console.warn('error: ' + err.message)
    })
})()