Last active
April 18, 2021 18:58
-
-
Save dispherical/933520a26e8db0af95b7709259f4011a to your computer and use it in GitHub Desktop.
How to fetch every package name on npm using node.js (requires a lot of memory)
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
console.log("Grabbing list of all packages from npm..."); | |
var request = require("sync-request"); | |
const cliProgress = require("cli-progress"); | |
var res = request("GET", "https://skimdb.npmjs.com/_all_docs"); | |
const fs = require("fs"); | |
var packages = JSON.parse(res.getBody("utf8")).rows; | |
var packagejson = { | |
name: "all-packages-in-a-json-file", | |
version: "1.0.0", | |
description: "", | |
main: "", | |
scripts: { | |
test: 'echo "Error: no test specified" && exit 1', | |
}, | |
keywords: [], | |
author: "", | |
license: "CC0", | |
dependencies: {}, | |
}; | |
console.log( | |
`Done. ${packages.length} packages were successfully grabbed from npm.` | |
); | |
console.log("Generating a package.json of the packages from npm."); | |
// create new progress bar | |
const b1 = new cliProgress.SingleBar({ | |
format: | |
"CLI Progress |" + | |
_colors.cyan("{bar}") + | |
"| {percentage}% || {value}/{total} Chunks || Speed: {speed}", | |
barCompleteChar: "\u2588", | |
barIncompleteChar: "\u2591", | |
hideCursor: true, | |
}); | |
b1.start(packages.length, 0, { | |
speed: "N/A", | |
}); | |
var i = 0; | |
while (i < packages.length) { | |
var package = packages[i].id; | |
packagejson.dependencies = Object.assign(packagejson.dependencies, { | |
[package]: "*", | |
}); | |
b1.increment(); | |
i++; | |
} | |
b1.stop(); | |
console.log(`Done. Saving to ${process.argv[2] || "allpackages.json"}`); | |
fs.writeFileSync( | |
process.argv[2] || "allpackages.json", | |
JSON.stringify(packagejson) | |
); |
rewrite that in c++ but it uses the system function
rewrite that in c++ but it uses the system function
sure why not
I got 16 gigs, should I give it a try?
no, JSON.stringify hates large objects, so I'm rewriting it in c++
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
I got 16 gigs, should I give it a try?