Mirror Node.js dist and download all npm versions for windows #script #npm #node #mirror

README.md

Scripts that mirror nodejs and npm

Note: The npm download script currently only download zip files

Improvement to do:

• Move to dedicated repo
• Add CLI tool for the npm and node download script
• Replace request lib with not deprecated one
• Download index.json by itself
• Install another formats of npm (not just zip)

Bugs

• The node install script is not installing scripts in win64 and win86 folder

download-all-npm-versions.js

// IMPORTANT:
// - Download this file from https://nodejs.org/dist/index.json and place it in the script directory with the name index.json
// - If you pass useNative: true then you should download `request`

// Download only ZIP files

var http = require('https');
var fs = require('fs');
const path = require('path');

let requestLib;

// This file is from https://nodejs.org/dist/index.json
const versionsFile = require('./index.json');

const DIST_DIR = path.join(__dirname, '../../npm');

const getAllVersions = () => {
	const versions = versionsFile.reduce((ver, curr) => {
		ver[curr.npm] = true;
		return ver;
	}, {});
	return Object.keys(versions).filter(Boolean);
};

const getDownloadLinkForVersion = (ver) => {
  // Dont use this https://github.com/npm/cli/archive/v${ver}.zip because it redirect to another url and this script not support it
  return `https://codeload.github.com/npm/cli/zip/v${ver}`
};

const downloadFile = ({dir, fileName, extension, url, overwrite = false, useNative = true} = {}) => {
	return new Promise((resolve, reject) => {
		const destPath = path.join(dir, fileName + (extension || ''));
		if(fs.existsSync(destPath)) {
			if(overwrite) {
				console.log('remove ' + destPath);
				fs.unlinkSync(destPath);
			} else {
				return resolve();
			}
		}
		var file = fs.createWriteStream(destPath);
		
		if(useNative) {
			var request = http.get(url, (response) => {
				response.pipe(file);
				file.on('finish', () => {
						file.close(resolve);  // close() is async, call cb after close completes.
					})
					.on('error', (err) => { // Handle errors
						fs.unlinkSync(destPath); // Delete the file async. (But we don't check the result)
						console.error('file on error', err);
						reject(err.message);
					});
			}).on('error', (err) => { // Handle errors
				fs.unlinkSync(destPath); // Delete the file async. (But we don't check the result)
				console.error('request on error', err);
				reject(err.message);
			});
		} else {
			requestLib = requestLib || require('request');
			let stream = requestLib({
				/* Here you should specify the exact link to the file you are trying to download */
				uri: url,
				headers: {
					'Accept': '*',
					'Accept-Encoding': 'gzip, deflate, br',
					'Cache-Control': 'max-age=0',
					'Connection': 'keep-alive',
					'Upgrade-Insecure-Requests': '1',
					'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'
				},
				/* GZIP true for most of the websites now, disable it if you don't need it */
				gzip: true
			})
				.pipe(file)
				.on('finish', () => {
					console.log(`The file is finished downloading.`);
					file.close(resolve);
					resolve();
				})
				.on('error', (error) => {
					reject(error);
				})
		}
	});
}

const failedDownload= [];
const downloadComplete = [];

const npmVers = getAllVersions();
const versionLength = npmVers.length;

Promise.all(
	npmVers.map(async (ver) => {
		const url = getDownloadLinkForVersion(ver);
		await downloadFile({dir: DIST_DIR, fileName: 'v' + ver, extension: '.zip', url})
			.then((res) => {
				downloadComplete.push(ver);
				console.log(`[${downloadComplete.length}/${versionLength}]: v${ver} finish download`, res);
			})
			.catch((err) => {
				console.error(`v${ver} failed download`, err);
				failedDownload.push(ver);
			})
	})
).then(() => {
	if(failedDownload.length > 0) {
		console.warn('Those versions failed', failedDownload);
	} else if(versionLength !== downloadComplete.length) {
		console.warn('Not all download completed', JSON.stringify(npmVers.filter(ver => !downloadComplete.includes(ver))));
	} else {
		console.log('No failed downloads');
	}
})
.catch(err => {
	console.error('crash');
	console.error('This version failed or didn\'t completed', JSON.stringify(npmVers.filter(ver => !downloadComplete.includes(ver))))
});

install-node.bat

REM Need wget
wget -nc -m -nH -e robots=off -np --convert-links -A="*.msi*, *index.html*, *-linux-x64.tar.gz*, *-linux-x86.tar.gz*, *-x64.msi*, *-x86.msi*, *.pkg*, *-linux-arm64.tar.gz*, *-linux-armv7l.tar.gz, *-darwin-x64.tar.gz*, *SHASUMS256.*, *npm-versions.txt*, *index.json*, *index.tab*" -X "*/v0*" http://nodejs.org/dist/