Download all files in an apache listing

download-apache-listing.js

#!/usr/bin/env node
const https = require("https");
const http = require("http");
const fs = require("fs");
const path = require("path");
const { JSDOM } = require("jsdom");

if (process.argv.length < 3) {
  console.error("Usage: node download-apache-listing.js <url>");
  process.exit(1);
}

const rootUrl = process.argv[2].endsWith("/")
  ? process.argv[2]
  : process.argv[2] + "/";
const urlObj = new URL(rootUrl);
const protocol = urlObj.protocol === "https:" ? https : http;

const visited = new Set();

function fetchListing(url) {
  return new Promise((resolve, reject) => {
    protocol
      .get(
        url,
        { rejectUnauthorized: false }, // ignore SSL
        (res) => {
          if (res.statusCode !== 200) {
            reject(
              new Error(`Request Failed: ${res.statusCode} ${res.statusMessage}`)
            );
            return;
          }

          let raw = "";
          res.on("data", (chunk) => (raw += chunk));
          res.on("end", () => resolve(raw));
        }
      )
      .on("error", reject);
  });
}

function downloadFile(url, localPath) {
  return new Promise((resolve, reject) => {
    const file = fs.createWriteStream(localPath);
    protocol
      .get(
        url,
        { rejectUnauthorized: false }, // ignore SSL
        (res) => {
          if (res.statusCode !== 200) {
            reject(
              new Error(`Download failed: ${res.statusCode} ${res.statusMessage}`)
            );
            return;
          }
          res.pipe(file);
          file.on("finish", () => file.close(resolve));
        }
      )
      .on("error", (err) => {
        fs.unlink(localPath, () => reject(err));
      });
  });
}

async function crawl(url, baseDir) {
  if (visited.has(url)) return;
  visited.add(url);

  console.log(`Listing: ${url}`);
  const html = await fetchListing(url);
  const dom = new JSDOM(html);
  const links = [...dom.window.document.querySelectorAll("a")];

  for (const link of links) {
    const href = link.getAttribute("href");
    if (
      !href ||
      href === "../" ||
      href.startsWith("?") ||
      href.startsWith("#")
    ) {
      continue;
    }

    const newUrl = new URL(href, url).toString();

    if (href.endsWith("/")) {
      // Directory
      const subDir = path.join(baseDir, href);
      await crawl(newUrl, subDir);
    } else {
      // File
      const localPath = path.join(baseDir, href);
      const dirName = path.dirname(localPath);
      if (!fs.existsSync(dirName)) {
        fs.mkdirSync(dirName, { recursive: true });
      }
      console.log(`Downloading: ${newUrl}`);
      await downloadFile(newUrl, localPath);
    }
  }
}

(async () => {
  const outDir = path.basename(urlObj.pathname) || "download";
  if (!fs.existsSync(outDir)) {
    fs.mkdirSync(outDir);
  }
  try {
    await crawl(rootUrl, outDir);
    console.log("✅ Done.");
  } catch (err) {
    console.error("❌ Error:", err);
  }
})();