Lists all files under a directory, recursively, and fast, in Scala

allFilesInDirectory.scala

      logger.info(s"Finding out how many files to process in $inputDirectory")

      // work around Java's dumb file IO methods
      def forEachFileToProcess(f: Path => Unit): Unit = {
        val filenamePattern = "^([a-f0-9]{40})\\.pdf$"r
        val visitor = new FileVisitor[Path] {
          override def visitFileFailed(file: Path, exc: IOException): FileVisitResult =
            FileVisitResult.SKIP_SUBTREE

          override def visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult = {
            file.getFileName.toString match {
              case filenamePattern(sha) if !seenPaperIds.contains(sha) => f(file)
              case _ => /* nothing */
            }

            FileVisitResult.CONTINUE
          }

          override def preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult =
            FileVisitResult.CONTINUE

          override def postVisitDirectory(dir: Path, exc: IOException): FileVisitResult =
            FileVisitResult.CONTINUE
        }

        Files.walkFileTree(Paths.get(inputDirectory), visitor)
      }

      val filesToProcess = {
        val buffer = mutable.Buffer[Path]()
        forEachFileToProcess { path => buffer += path }
        buffer.par
      }

Java's old IO has some functions for this too which are easier to use, but they either have problems with error handling, are super slow once you have more than a trivial number of files, or both.