Skip to content

Instantly share code, notes, and snippets.

@tirkarthi

tirkarthi/build-top.py

Last active March 31, 2018 12:46
Show Gist options
  • Save tirkarthi/846f9d2fd87c90463c5ca9cf99f3d44d to your computer and use it in GitHub Desktop.
Save tirkarthi/846f9d2fd87c90463c5ca9cf99f3d44d to your computer and use it in GitHub Desktop.
Download ZIP
Dependants list for 49305 issue
Raw
build-top.py
import json
from collections import OrderedDict
from pymongo import MongoClient
deps = {}
connection = MongoClient()
database = connection.crates
collection = database.crates_data
with open("files.txt") as f:
for line in f:
line = line.strip()
module_name = line.split("/")[-1]
count = collection.find({"deps": {"$elemMatch": {"name": module_name, "kind": "normal"}}}).count()
deps[module_name] = count
print(json.dumps(OrderedDict(sorted(deps.items(), key=lambda x: x[1], reverse=True)[:100]), indent=1))
Raw
rust-49305.txt
# stdweb as dependency
> db.crates_data.distinct("name", {deps: {$elemMatch : {name: "stdweb", "kind": "normal"}}}, {"name": 1})
[
"rand",
"hobofan_stdweb_logger",
"bosh_compiler",
"webgl",
"webcomponent",
"screeps-game-api",
"virtual_view_dom",
"genact",
"livesplit-hotkey",
"cpal",
"dominator",
"papito_dom",
"papito",
"squark-stdweb",
"yew"
]
# http as dependency
> db.crates_data.distinct("name", {deps: {$elemMatch : {name: "http", "kind": "normal"}}}, {"name": 1})
[
"thruster",
"shio",
"hyper",
"json-api",
"kl-http",
"luminal-router",
"luminal-handler",
"actix-web",
"actix-redis",
"h2",
"ruma-api",
"http-status-print",
"sockjs",
"mio_httpc",
"mini_http",
"reset-router",
"simple-server",
"ni_ce",
"chttp"
]
# rand as dependency
> db.crates_data.distinct("name", {deps: {$elemMatch : {name: "rand", kind: "normal"}}}, {"name": 1})
[
"rdrand",
"rdedup-lib",
"dsfmt-rs",
"ndarray-rand",
"ndarray-linalg",
"ndarray-numtest",
"markov-chain",
"markov",
"math_traits",
"maze",
"matrixnum",
"maidsafe_client",
"maidsafe_types",
"maidsafe_utilities",
"marmoset",
"mallumo",
"mackerel_plugin",
"argmin",
"arendur",
"aries",
"tcpproxy",
"graph-generators",
"grid-sim",
"grafen",
"o2lsh",
"mcts",
"objectid",
"object-alloc-test",
"rfyl",
"ice_core",
"plutchik",
"planetkit",
"pleco",
"pleco_engine",
"plumber",
"lather",
"lars",
"lazysort",
"largo",
"lamport_sigs",
"lapjv",
"lafont",
"vec-vp-tree",
"vegas-rs",
"vegas-lattice",
"theban_interval_tree",
"thrussh-agent",
"thrussh_server",
"threshold-secret-sharing",
"theca",
"pgen",
"brids",
"ncollide_math",
"ncollide_testbed2d",
"ncollide_testbed3d",
"ncollide_utils",
"cfrp",
"offwall",
"aws_kms_crypt",
"x25519-dalek",
"jobsteal",
"jobserver",
"cards",
"catfs",
"cargo-cov",
"cargo-registry",
"cargo-make",
"cargo-brew",
"cargo-contribute",
"cargo-package-recompile",
"captcha",
"carp",
"caffrey",
"caesarlib",
"cage",
"calco",
"castle-game",
"dynamo",
"dyon",
"shadowsocks",
"shadowsocks-rust",
"shadowsocks-rs",
"shuffled-iter",
"shaman",
"shamir",
"shunting",
"hyper-dns",
"hyperloglog",
"hyper-multipart-rfc7578",
"hyper-multipart-rfc7578-mem-uploads",
"rads",
"random_utils",
"rand-pop",
"randnum",
"random-pool",
"rand-mersenne-twister",
"random_color",
"rando",
"random-wheel",
"random-world",
"random_derive",
"random-names",
"rand-extra",
"random_choice",
"rand-distributions",
"rayon-core",
"raider-server",
"radix_sort",
"ramp",
"raft",
"rahashmap",
"rasputin",
"ranagrams",
"rabbiteer",
"outoforderfs",
"tftp_server",
"rojo",
"routing",
"RobotS",
"roulette-wheel",
"rouler",
"roulette",
"rofl",
"rogcat",
"rolldice",
"rome",
"rooster",
"rotor-dns",
"rouille",
"rocksd",
"email",
"emcee",
"emerald-core",
"emerald-rs",
"fuss",
"fuzzmutator",
"futures-backoff",
"fujisaki_ringsig",
"exar",
"exar-testkit",
"exoquant",
"extprim",
"excuse",
"exponential-decay-histogram",
"executors",
"exonum-btc-anchoring",
"exonum",
"utah",
"utp2",
"phf_generator",
"pitch_calc",
"pippin",
"pijul",
"pick-one",
"rsmath",
"rs_poker",
"rsass",
"rsbx",
"rsgenetic",
"rsmq",
"rshare",
"rspotify",
"rsrl",
"rsdb",
"indy-crypto",
"indy",
"indy-sdk",
"insult",
"indradb-lib",
"indradb",
"instrument",
"fera-ext",
"fera-graph",
"fe_session",
"s3lsio",
"udpproxy",
"ggp-rs",
"adivon",
"adapton-lab",
"addressable-pairing-heap",
"abstract-ns",
"bson",
"jsonrpc-ipc-server",
"json-job-dispatch",
"devp2p-secp256k1",
"devp2p",
"delix",
"dementia",
"deckofcards",
"skiplist",
"sandbox-ipc",
"sarkara",
"sapling-crypto",
"safe_vault",
"safe_app",
"safe_authenticator",
"safe_client",
"safe_network_common",
"safe_core",
"sabisabi",
"sanakirja",
"gotham",
"good_stv",
"uuid_v1",
"uuid",
"uuid_v1_variant",
"uuid1",
"probe-c-api",
"printpdf",
"promiser",
"preserve",
"proptest",
"prophet",
"prototty_elements",
"programinduction",
"primg",
"primitive",
"elma",
"eligos",
"eliza",
"elmesque",
"vtebench",
"kyber",
"wordrand",
"woot",
"worley-noise",
"sfmt",
"waveform_space",
"walmart_partner_api",
"wamp",
"quicksilver",
"quickercheck",
"quickcheck",
"quickcheck_seedable",
"quantum",
"xkpwgen",
"xkcd",
"xkcdpass",
"hifitime",
"histogram-sampler",
"rcache",
"rcurses",
"glit",
"glm_color",
"gluon",
"bn-plus",
"yobicrypto",
"pwrs",
"pwgenr",
"pwhash",
"twist",
"twisty",
"twox-hash",
"twapi",
"fisher",
"fixed-hash",
"file_hashmap",
"filesystem",
"boringauth",
"boolean_expression",
"boiler",
"alcibiades",
"algs4",
"aliasmethod",
"alumina",
"bencode-decoder",
"beamdpr",
"bellman",
"fanta-cli",
"faktory",
"faker",
"fake",
"fasthash",
"fail",
"fnorder",
"amber",
"bigwise",
"bitcoin",
"bins",
"bip_utracker",
"bip_handshake",
"bip_util",
"bip_dht",
"bip_select",
"bismuth",
"bip39",
"websocket-vi",
"websocket",
"weakrand",
"weldmock",
"webhook-listen",
"webdriver_client",
"wee-matrix",
"wechat",
"ultra",
"ulid",
"ludomath",
"newhope",
"neural_network",
"neuroflow",
"neil",
"netsim",
"asmuth_bloom_secret_sharing",
"astrup",
"juice",
"juggernaut",
"jumphash",
"flock",
"flattiverse_connector",
"cifar_10_loader",
"ciruela",
"zipkin",
"zipf",
"xero",
"acyclic-network",
"acgmath",
"acacia",
"actix-web",
"actix_derive",
"actix-redis",
"ykcryptsetup",
"lru2",
"oxcable",
"oxide-auth",
"rpc-perf",
"rpcperf_workload",
"rpwg",
"mongodb",
"mongors",
"morocco",
"monto",
"monotone",
"moby-name-gen",
"moonlander-gp",
"mockup_hal",
"mockito",
"cervisia",
"cernan",
"cellular_maps",
"murmurhash64",
"multipart-async",
"multirust-rs",
"multipart",
"loaded_dice",
"lolcat",
"lonlat_bng",
"lockfreehashmap",
"zero_sum",
"white-balance",
"kademlia",
"karplus",
"karma",
"pushid",
"pumpkindb_engine",
"pumpkin",
"pumpkindb_mio_server",
"la",
"ai",
"bn",
"rp",
"ff",
"ge",
"b2",
"ga",
"fw",
"nn",
"ws",
"s2",
"xoroshiro",
"xoroshiro128",
"xorshift",
"xor_name",
"persy",
"petname",
"petri",
"octavo-crypto",
"bcrypt-bsd",
"bcrypt",
"vpncloud",
"pcg_rand",
"egg-mode",
"suppositions",
"subotai",
"surt",
"sunk",
"sudoku",
"scarecrow",
"scram",
"schroedinger_box",
"scalable_cuckoo_filter",
"scp03",
"score",
"scirust",
"screenruster-saver-hacks",
"screenruster",
"screeps-api",
"screenruster-saver-fractal",
"mli_mep",
"wfst",
"gnunet",
"kolmogorov_smirnov",
"evm-stateful",
"evco",
"evzht9h3nznqzwl",
"teardown_tree___treap",
"teardown_tree",
"textnonce",
"terra",
"temp_utp",
"tempfile",
"tempdir",
"tempfile-fast",
"test-data-generation",
"teleecho",
"easyfibers",
"ruma-identifiers",
"ruplace",
"rubefunge-93",
"run_script",
"ruschankunsan",
"rumqtt",
"rulid",
"rurel",
"rusq",
"rustun",
"rust_keylock",
"rust_redlock",
"rusticsom",
"rusty_secrets",
"rustils",
"rustlearn",
"rusty-leveldb",
"rust-kpdb",
"rustml",
"rust-life",
"rusty-santa",
"rustplacements",
"rust_sodium",
"rusty_express",
"rustix-bl",
"rustneat",
"rust_sodium-sys",
"rustracing",
"rusty_dashed",
"rustr",
"rust-oauth-proxy",
"rust-paillier",
"rust-crypto",
"rust-gm-paillier",
"rust-install",
"rustracing_jaeger",
"rusty-machine",
"rust-monster",
"dars",
"dagon",
"dalek-rangeproofs",
"dalek-credentials",
"ntrumls-plus",
"ntree",
"crack",
"cratedb",
"criterion-stats",
"crypto_vault",
"crow_engine",
"crust",
"aes-stream",
"aerospike",
"tungstenite",
"turbo",
"tupm",
"turtle",
"ttyaskpass",
"apint",
"apns",
"ergo_sys",
"erl_dist",
"trawler",
"transport",
"tracery",
"traceroute",
"treap",
"traffic_generator",
"treez",
"trip",
"travelling_salesman",
"trust-dns-proto",
"trust-dns-server",
"trust-dns",
"idcontain",
"ksuid-cli",
"ksuid",
"rtrie",
"rtop",
"dice-me",
"diceware-rs",
"diceware",
"diceware-gen",
"differential-evolution",
"ditto",
"dilithium",
"cgmath",
"gelf",
"gear",
"genetic_planner",
"genetic-files",
"genevo",
"genact",
"optimization",
"opentracingrust_zipkin",
"opentracingrust",
"open189",
"openai",
"opcua-core",
"opcua-server",
"opcua-types",
"oprec",
"tic_tac_toe",
"timi",
"time_calc",
"timeout",
"bui-backend",
"bulletinboard",
"buildchain",
"snow",
"bandit",
"bakervm",
"bare-websocket",
"basiccountminsketch",
"badtouch",
"backgammon",
"backoff",
"hematite_server",
"hex2d-dpcext",
"hex2d",
"hecate",
"hexe",
"hexe_core",
"hetu",
"csrf",
"dungen",
"dudect-bencher",
"mqtt",
"mqttc",
"gaol",
"game2048",
"game-2048",
"gameai",
"gamer",
"conductor",
"conc",
"concurrent",
"concurrent-hash-map",
"constrained",
"ConExpression",
"cognitive-exhibitor",
"content",
"coap",
"cobalt",
"compress",
"competitive-programming-rs",
"collision",
"collection",
"count-min-sketch",
"combid",
"core_collections",
"cool_faces",
"convex_hull",
"conway",
"conway-rs",
"orca",
"orion",
"tql_macros",
"td_revent",
"lindenmayer-system",
"libreauth",
"librandtest",
"lightning",
"lipsum",
"libstoragemgmt-rust",
"libsecp256k1",
"libsodium_seeded_prng",
"libnetkeeper",
"libmodbus-rs",
"libmaj",
"libmath",
"libgo",
"libicmp",
"linxal",
"libcoinche",
"lib_dice",
"littlewing",
"libpijul-compat",
"libpijul",
"ed25519-dalek",
"voronoi",
"vorbis",
"voc-perturb",
"voxel_worldgen",
"voidmap",
"vosealias",
"vox_box",
"ipc-channel",
"iptrap",
"wyrm",
"http2parse",
"ethcore-bigint",
"ethereum-bigint",
"etcommon-bigint",
"etclient",
"framp",
"free-space-wipe",
"valora",
"vape",
"vanityhash",
"hdbconnect",
"self_encryption",
"sema",
"servo-websocket",
"service_discovery",
"session",
"seabloom",
"serial-key",
"sentinel",
"secret-service",
"secp256k1-plus",
"secp256k1",
"secp256k1-test",
"secure-session",
"mpid_messaging",
"blissb",
"bluenoisers",
"bloomfilter",
"bloom_filter",
"bloom-server",
"blossom",
"r",
"encryptfile",
"enimda",
"ensync",
"engine-io",
"engiffen",
"winauth",
"num-bigint",
"numeric",
"cpp_codegen",
"jeepers",
"ozone",
"ktm5e-dice",
"ktensor",
"avr-vm",
"avl_tree",
"curve25519-dalek",
"curs",
"curtain",
"cuckoofilter",
"srt-transport",
"dribble",
"hprose",
"msgpacknet",
"handoff_counter",
"harvey-holt",
"hangman",
"hashmap2",
"hal-ml",
"hate",
"hawk",
"hacl-star",
"haikunator",
"hamt-rs",
"hamt-sync",
"pnetlink",
"poisson",
"powersoftau",
"polk",
"portier_broker",
"portunes",
"pokerhandrange",
"poke-a-mango",
"pokemon",
"postio",
"postgres-cursor",
"postgres-protocol",
"rncryptor",
"bdrck",
"sophon-wordlist",
"sovrin-client",
"son_of_grid_engine",
"sodalite",
"sorter",
"sorting",
"sodiumoxide_extras",
"sozu",
"sozu-lib",
"sozuctl",
"source-map-mappings",
"sockjs",
"oil-lang",
"vgmdb",
"irc-bot",
"iron-dsc-csrf",
"iron-sessionstorage2",
"iron-arrow",
"iron-sessionstorage",
"sputnikvm-stateful",
"sputnikvm-dev",
"sprite-gen",
"spake2",
"spanquist",
"spacebar",
"spaces",
"spherical-cow",
"wc-lock",
"keygen",
"kernel_density",
"ketos",
"domain",
"doggo",
"clingo",
"clocked-dispatch",
"clipping",
"closed01",
"clowder",
"noise-rust-crypto",
"noise",
"noir",
"audact",
"autograd",
"sdl2",
"mkserial",
"zalgo",
"parking_lot_core",
"passwors",
"passgen",
"parity-wordlist",
"panty",
"patricia-trie",
"pagecache",
"pam_groupmap",
"pairing",
"pastebin",
"paillier",
"pact_verifier_cli",
"pacthash",
"pact_mock_server_cli",
"pact_matching",
"mindtree_utils",
"milagro-crypto",
"minesweeper",
"mildew",
"microfacet",
"microsalt",
"microbit",
"mio_httpc",
"mimir",
"mief",
"minimax",
"MBrane",
"reql-io",
"retry",
"redis-cluster",
"redis-client",
"regex_generate",
"revonet",
"reservoir",
"repose",
"resolve",
"resource_proof",
"redox_users",
"redox",
"redlock",
"redlock-rs",
"lsystems",
"pf_sandbox",
"sqs-service-helper",
"squark",
"byte-sequence",
"nv-card",
"merkle_test",
"merkle_light",
"memcached-rs",
"mersenne_twister",
"message_verifier",
"melon",
"meep",
"median_three_quicksort",
"memsec",
"metaheuristics",
"rlpx",
"tfs",
"zkp",
"zdd",
"mtl",
"mob",
"mon",
"e2r",
"evo",
"gst",
"gol",
"gip",
"glm",
"abc",
"dpt",
"dht",
"d20",
"ddp",
"dsp",
"utp",
"oic",
"rkm",
"rpg",
"rix",
"rrt",
"rvs",
"rng",
"rsl",
"sdl",
"sc2",
"srv",
"xsv",
"lfa",
"lal",
"bfc",
"bnf",
"bvh",
"igd",
"i2p",
"ire",
"px8",
"p2p",
"xstream",
"urdf-viz",
"urdict",
"sincere",
"sidekiq",
"sidekiq-rs",
"sidh",
"simple_gaussian",
"simple",
"simple_stats",
"simple_lsystems",
"simple_ea",
"simpleflake",
"simple-cards",
"yyid",
"oozie",
"oozz",
"nat_traversal",
"navigation",
"nanoid",
"nanopow-rs",
"nano-vanity",
"nanocurrency-peering",
"nalgebra",
"names",
"nannou",
"nats",
"nats_client",
"yubibomb",
"yubihsm",
"yubikey",
"yubico",
"starwars-names",
"stemjail",
"stable-skiplist",
"striple",
"string_generator",
"stellar_vanity",
"stamm",
"stdcli",
"statsd",
"static-http-cache",
"statistical",
"statrs",
"steam-crypto",
"stdx",
"stun",
"stun3489",
"stag",
"implicit",
"image-stream",
"imageproc",
"task_kit",
"tacobell",
"cdrs",
"tozny_auth",
"tokio-by-hand",
"tokio-linux-aio",
"tokio-http2",
"tokio-utp",
"tokio-retry",
"tokio-ping",
"tokio-proto",
"tokio-threadpool",
"onionsalt",
"ryb_game",
"tsunami",
"syncer",
"synth",
"djinn",
"djangohashers",
"rzbackup",
"iodyn",
"iomrascalai",
"ios7crypt",
"guessing_game",
"mtcp",
"oauthcli",
"oauth1",
"oauth-client",
"oauth-client-fix",
"nitro",
"nihdb",
"nickel_cookies",
"git-sql",
"git-mix",
"gimei",
"gist-i",
"chan",
"chanvese",
"chess-move-gen",
"chaos",
"chip8_emu",
"chip8_vm",
"chipper",
"chimper",
"forkjoin",
"fortunelike",
"fortune",
"fountaincode",
"fountain_codes",
"ezra",
"tk-pool",
"tk-http",
"tk-carbon",
"slab-alloc",
"slider",
"sloword2vec",
"slowloris",
"dbus-bytestream",
"swindon",
"leaf",
"zcash-vanity",
"pdatastructs",
"knock"
]
Raw
scraper.py
# Generate files.txt with `find . -type f > files.txt`
import json
from pymongo import MongoClient
with open("files.txt") as files:
client = MongoClient()
db = client.crates # crates db
items = []
for path in files:
path = path.strip()
if path.endswith("txt") or path.endswith("json") or path.endswith("py"):
continue
with open(path) as f:
# Last line for latest dependency
content = f.readlines()[-1].strip()
try:
items.append(json.loads(content))
except Exception as e:
print("Bad file ", path)
collection = db.crates_data # crates_data as collection
collection.insert_many(items)
Raw
top-100.json
{
"serde": 2081,
"libc": 1768,
"serde_derive": 1542,
"serde_json": 1488,
"log": 1431,
"clap": 1063,
"hyper": 1025,
"rand": 944,
"lazy_static": 892,
"regex": 753,
"byteorder": 738,
"futures": 705,
"winapi": 680,
"error-chain": 621,
"rustc-serialize": 620,
"url": 614,
"chrono": 578,
"time": 533,
"tokio-core": 446,
"bitflags": 406,
"mime": 390,
"clippy": 388,
"env_logger": 380,
"toml": 353,
"num": 346,
"yup-oauth2": 325,
"quote": 295,
"syn": 294,
"itertools": 261,
"nom": 260,
"failure": 257,
"reqwest": 256,
"num-traits": 235,
"uuid": 229,
"base64": 209,
"docopt": 191,
"tokio-io": 188,
"hyper-rustls": 175,
"quick-error": 174,
"rust-crypto": 174,
"bytes": 173,
"getopts": 163,
"strsim": 163,
"image": 163,
"yup-hyper-mock": 154,
"num_cpus": 151,
"nix": 147,
"openssl": 132,
"tempdir": 130,
"flate2": 127,
"iron": 127,
"ansi_term": 125,
"mio": 124,
"kernel32-sys": 121,
"rayon": 119,
"walkdir": 118,
"glob": 116,
"rusoto_core": 113,
"fnv": 108,
"term": 107,
"bincode": 105,
"slog": 103,
"xml-rs": 91,
"semver": 90,
"ring": 85,
"crossbeam": 81,
"native-tls": 81,
"glib-sys": 79,
"hex": 76,
"failure_derive": 72,
"nalgebra": 72,
"smallvec": 71,
"serde_macros": 71,
"curl": 71,
"csv": 71,
"serde_yaml": 70,
"git2": 70,
"protobuf": 69,
"sha2": 68,
"futures-cpupool": 68,
"hyper-tls": 67,
"cortex-m": 67,
"colored": 66,
"structopt": 66,
"hyper-native-tls": 62,
"gobject-sys": 62,
"yaml-rust": 62,
"enum_primitive": 60,
"parking_lot": 60,
"vcell": 59,
"cfg-if": 57,
"generic-array": 57,
"cgmath": 56,
"ndarray": 54,
"errno": 54,
"bare-metal": 54,
"encoding": 54,
"libimagerror": 53,
"memmap": 53,
"digest": 52
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment