markdown url downloader

downloader.js

const fs = require("fs");
const fetch = require("node-fetch");
const debug = require("debug")("staticFolio:fetch-content");
const chalk = require("chalk");
const mkdirp = require("mkdirp");
const path = require("path");
const dotenv = require("dotenv").config();

const downloadMarkdown = async (url) => {
	url = decodeURI(url);

	// fetch the content in async. await the response immediately
	const response = await fetch(url);

	if (response.status != 200) {
		debug(chalk.red(`Error fetching file: ${response.status}`));
	}

	// return the markdown text
	return await response.text();
};

/**
 * Downloads array of urls pointing to markdown
 * @param {Array} urls - array of urls to markdown content
 * @example fetchContent (["https://raw.website.com/file.md", "https://raw.website.com/anotherFile.md"])
 */
const fetchContent = async (urls) => {
	for (const url of urls) {
		const markdown = downloadMarkdown(url);
		const urlParts = new URL(url);
		// debug(urlParts);

		// get the path to write to based on the path of the URL. This generally ensures that related files go in similar locations
		// for example: https://raw.githubusercontent.com/RolandWarburton/knowledge/master/programming/Principles of OOP.md
		// resolves to ROOT/uploads/RolandWarburton/knowledge/master/programming/Principles of OOP.md
		const writePath = path.resolve(
			__dirname,
			"uploads",
			decodeURI(urlParts.pathname).substr(1, urlParts.pathname.length)
		);

		// debug(path.parse(urlParts.pathname));
		// debug(writePath);

		// debug(process.env.ROOT);
		// debug(writePath);
		// debug(decodeURI(urlParts.pathname));
		// debug(path.parse(writePath).dir);

		mkdirp(path.parse(writePath).dir)
			.then(async () => {
				fs.writeFileSync(writePath, await markdown);
			})
			.catch((err) => {
				debug(err);
			});
	}
};

module.exports = fetchContent([
	"https://raw.githubusercontent.com/RolandWarburton/knowledge/master/programming/Principles of OOP.md",
]);