Asynchronous node js file downloader with progress bar

downloader.js

#!/usr/bin/node
const axios = require("axios");
const progress = require("cli-progress");
const fs = require("fs");
const async = require("async");

// https://stackoverflow.com/questions/10420352/converting-file-size-in-bytes-to-human-readable-string
const filesize = (a, b, c, d, e) => {
  return (
    ((b = Math),
    (c = b.log),
    (d = 1e3),
    (e = (c(a) / c(d)) | 0),
    a / b.pow(d, e)).toFixed(e ? 2 : 0) +
    " " +
    (e ? "kMGTPEZY"[--e] + "B" : "Bytes")
  );
};

/**
 * https://github.com/axios/axios#request-config
 * https://github.com/npkgz/cli-progress/tree/master#options-1
 */
const _download = async (url) => {
  const prog = new progress.Bar({
    barCompleteChar: "━",
    barInCompleteChar: "-",
    fps: 10,
    stream: process.stdout,
    barsize: 30,
    stopOnComplete: false,
    clearOnComplete: false,
    format:
      "{filename}: {bar} {percentage}% | {current}/{totalMax} | ETA: {eta}s",
  });
  const filename = url.split("/").slice(-1)[0];
  const stream = fs.createWriteStream(filename);
  return new Promise(async (resolve, reject) => {
    await axios({
      url: url,
      method: "GET",
      headers: {
        "User-Agent":
          "Mozilla/5.0 (Linux; Android 11; Infinix X6810) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Mobile Safari/537.36",
      },
      responseType: "stream",
    }).then((response) => {
      let current = 1;
      let total = parseInt(response.headers["content-length"]);
      prog.start(total, 0);
      prog.update({ filename: filename, totalMax: filesize(total) });
      response.data
        .on("data", (chunk) => {
          current += chunk.length;
          prog.increment(chunk.length);
          prog.update({ current: filesize(current) });
        })
        .pipe(stream);
      response.data.on("error", (err) => {
        prog.stop();
        reject(err);
      });
    });
    stream.on("finish", () => {
      prog.stop();
      resolve(`file saved as: ${filename}`);
    });
  });
};

/**
 * credit: @gist/motebaya
 * 09-26-2023
 */

here are 3 method for call it that i've research.

1. Call with async.queue, here you can set how much task can run in concurrency, and maybe this like max_workers in ThreadPoolExecutor python?

const asyncQueue = async (urls) => {
  console.log("-".repeat(10), "Called with async.queue", "-".repeat(10));
  const queue = async.queue(async (task, callback) => {
    await _download(task.url);
    callback();
  }, 1); // concurrency limit

  // add items to queue
  urls.forEach((url) => {
    queue.push({ url });
  });

  // asign to queue
  const start_time = Date.now();
  queue.drain((err) => {
    const duration = ((Date.now() - start_time) / 1000).toFixed(2);
    console.log(`Tasks completed in: ${duration} seconds`);
  });
};

asyncQueue([....an array of urls]);

Demo:

2. Call with Promise.all. using this are more so fast, but if all task run in concurrency the progress bar are being overwritten.

const asyncPromise = async (urls) => {
  console.log("-".repeat(10), "Called with Promise.all", "-".repeat(10));
  const start_time = Date.now();
  await Promise.all(
    urls.map(async (url) => {
      await _download(url);
    })
  );
  const duration = ((Date.now() - start_time) / 1000).toFixed(2);
  console.log(`Tasks completed in: ${duration} seconds`);
};

asyncPromise([....an array of urls]);

Demo:

3. Call with Iterating over an array of urls, whatever you can using .forEach, 'while, for (...) or .etc. but this method more than slow between 2 method before.

const Execute = async (urls) => {
  console.log(
    "-".repeat(10),
    "Called with iterate array of urls",
    "-".repeat(10)
  );
  const start_time = Date.now();
  for (const url of urls) {
    await _download(url);
  }
  const duration = ((Date.now() - start_time) / 1000).toFixed(2);
  console.log(`Tasks completed in: ${duration} seconds`);
};

Execute([....an array of urls]);

Demo:

---

Using Promise.all when you need more fast download because this method run all task in concurrency, if you've to much url to download e.g: 1000-2000 maybe will be consume more system resources. i didn't try it yet.

Using: async.queue you can set by self how much concurrency limit.