Skip to content

Instantly share code, notes, and snippets.

@graphicore
Last active May 25, 2018 14:22
Show Gist options
  • Save graphicore/7d52293d4132262a262c4ec747111262 to your computer and use it in GitHub Desktop.
Save graphicore/7d52293d4132262a262c4ec747111262 to your computer and use it in GitHub Desktop.
// jshint esversion:6
const NodeGit = require('nodegit')
, path = require('path')
, MAX_PARALLEL_JOBS = 8
, REPOSITORY_BASE_PATH = '/tmp/node_git_bug_1495'
;
/**
* Expecting typical github urls here, like: "https://github.com/username/projectname"
* returns REPOSITORY_BASE_PATH + "/username_projectname"
*/
function getRepoPath(remoteUrl) {
let dir = remoteUrl.split('/')
// remove empty entries, if the url ended with a slash
.filter(x=>x)
.slice(-2).join('_');
return path.join(REPOSITORY_BASE_PATH, dir);
}
/**
* Get remote with name `remoteName`. If the remote does not exist it
* is created and `remoteUrl` is used.
*/
function get_remote(repo, remoteName, remoteUrl) {
return NodeGit.Remote.create(repo, remoteName, remoteUrl)
.then(null, err=> {
if(err.errno !== NodeGit.Error.CODE.EEXISTS)
throw err;
return repo.getRemote(remoteName);
});
}
function gitJob(remoteUrl) {
// get the repository
let repositoryPath = getRepoPath(remoteUrl);
return NodeGit.Repository.init(repositoryPath, /*isBare:*/1)
// get/create the remote
.then(repo=>get_remote(repo, /*remoteName:*/'upstream', remoteUrl))
// fetch the branch
.then(remote=>remote.fetch('master'));
}
/**
* We use `workerId` to log and get an idea of the flow of execution.
* There's probably not much more to get from this information.
*/
function runJsWorker(workerId, queue, [success, fail, total]) {
if(!queue.length) {
// Out of jobs; This worker is done, don't return another promise.
console.log('worker-id', workerId, 'no jobs left, terminating');
return true;
}
let job = queue.shift();
console.log('worker-id', workerId, 'dispatching job', job
, 'previous success:', success,'fail:', fail, 'total:', total);
return gitJob(job)
.then(()=>{
console.log('worker-id', workerId, 'done job', job);
return [success+=1, fail, total+=1];
}
, err=>{
console.error('worker-id', workerId, 'failed job', job
, 'with', err);
// don't end the worker (by re-raising)
return [success, fail+=1, total+=1];
}
)
// repeat
.then(runJsWorker.bind(null,workerId, queue));
}
function main(queue) {
var jsWorkers = [];
console.log('Starting', MAX_PARALLEL_JOBS, 'js "workers", for'
, queue.length, 'jobs.');
for(let i=0;i<MAX_PARALLEL_JOBS;i++)
jsWorkers.push(runJsWorker(i, queue, [0, 0, 0]));
return Promise.all(jsWorkers).then(
results=>{
console.log('DONE', results.length, 'workers!');
return 0;
}
, err=>{
console.error('FAILED:', err);
return 1;
}
);
}
var queue = [
"https://github.com/googlefonts/abeezee.git"
, "https://github.com/m4rc1e/Abel"
, "https://github.com/googlefonts/MooniakFontsGFVersion"
, "https://github.com/cyrealtype/Adamina.git"
, "https://github.com/adobe-fonts/adobe-blank"
, "https://github.com/huertatipografica/Alegreya"
, "https://github.com/huertatipografica/Alegreya-Sans"
, "https://github.com/AlessioLaiso/aleo"
, "https://github.com/jmsole/alfaslab"
, "https://github.com/cyrealtype/Alice"
, "https://github.com/cyrealtype/Alike.git"
, "https://github.com/antonxheight/Allan"
, "https://github.com/googlefonts/amaranth"
, "https://github.com/googlefonts/AmaticSC.git"
, "https://github.com/cyrealtype/amethysta"
, "https://github.com/alif-type/amiri"
, "https://github.com/etunni/Amita.git"
, "https://github.com/TypeNetwork/fb-Amstelvar"
, "https://github.com/googlefonts/anaheimFont.git"
, "https://github.com/googlefonts/AntonFont.git"
, "https://github.com/vernnobile/antonioFont/"
, "https://github.com/etunni/arapey"
, "https://github.com/Omnibus-Type/ArchivoBlack"
, "https://github.com/Omnibus-Type/ArchivoNarrow"
, "https://github.com/alexeiva/Arsenal"
, "https://github.com/cyrealtype/Artifika"
, "https://github.com/antonxheight/Arvo"
, "https://github.com/etunni/Arya.git"
, "https://github.com/Omnibus-Type/Asap"
, "https://github.com/Omnibus-Type/AsapCondensed"
, "https://github.com/EbenSorkin/Asar.git"
, "https://github.com/EbenSorkin/Atomic-Age.git"
, "https://github.com/cadsondemak/athiti"
, "https://github.com/cyrealtype/Aubrey"
, "https://github.com/etunni/average"
, "https://github.com/alexeiva/badscript"
, "https://github.com/Omnibus-Type/Bahiana"
, "https://github.com/agoodfeelingco/bakbak"
, "https://github.com/googlefonts/bangers"
, "https://github.com/jpt/barlow"
, "https://github.com/Omnibus-Type/Barrio"
, "https://github.com/EbenSorkin/Basic.git"
, "https://github.com/shinntype/bellefair"
, "https://github.com/etunni/belleza"
, "https://github.com/googlefonts/BevanFont.git"
, "https://github.com/10four/Bhavuka.git"
, "https://github.com/typeoff/biryani"
, "https://github.com/solmatas/Bitter"
, "https://github.com/cyrealtype/Bonbon"
, "https://github.com/cyrealtype/Brawler"
, "https://github.com/djrrb/Bungee"
, "https://github.com/impallari/Cabin.git"
, "https://github.com/impallari/CabinSketch"
, "https://github.com/Gue3bara/Cairo"
, "https://github.com/anexasajoop/cambay.git"
, "https://github.com/GNOME/cantarell-fonts"
, "https://github.com/googlefonts/CardoFont"
, "https://github.com/VanillaandCream/Catamaran"
, "https://github.com/etunni/Changa"
, "https://github.com/etunni/changa-one"
, "https://github.com/Omnibus-Type/Chivo"
, "https://github.com/cadsondemak/chonburi.git"
, "https://github.com/vernnobile/CodaFont"
, "https://github.com/alexeiva/comfortaa"
, "https://github.com/CatharsisFonts/Cormorant.git"
, "https://github.com/librefonts/courgette"
, "https://github.com/vernnobile/coustardFont"
, "https://github.com/alexeiva/cuprum"
, "https://github.com/googlefonts/CutiveFont.git"
, "https://github.com/impallari/DancingScript"
, "https://github.com/meirsadan/david-libre"
, "https://github.com/EbenSorkin/Dekko.git"
, "https://github.com/vernnobile/DhyanaFont"
, "https://github.com/ossobuffo/didact-gothic"
, "https://github.com/elms-/Dinah"
, "https://github.com/etunni/diplomata"
, "https://github.com/khaledhosny/sahl-naskh"
, "https://github.com/googlefonts/MuktaGFVersion.git"
, "https://github.com/Gue3bara/El-Messiri"
, "https://github.com/impallari/Encode-Sans"
, "https://github.com/NDISCOVER/Exo-1.0"
, "https://github.com/etunni/fauna-one"
, "https://github.com/cyrealtype/Federant.git"
, "https://github.com/cyrealtype/Federo"
, "https://github.com/googlefonts/FiraGFVersion.git"
, "https://github.com/googlefonts/francoisoneFont.git"
, "https://github.com/etunni/Gabriela"
, "https://github.com/etunni/gilda-display"
, "https://github.com/etunni/glegoo.git"
, "https://github.com/etunni/graduate"
, "https://github.com/googlefonts/GreatVibesFont"
, "https://github.com/googlefonts/Homenaje.git"
, "https://github.com/cyrealtype/Iceberg"
, "https://github.com/cyrealtype/Iceland"
, "https://github.com/etunni/imprima"
, "https://github.com/cyrealtype/Inconsolata"
, "https://github.com/googlefonts/indieflower.git"
, "https://github.com/clauseggers/Inknut-Antiqua.git"
, "https://github.com/cadsondemak/itim.git"
, "https://github.com/cyrealtype/Jacques-Francois"
, "https://github.com/cyrealtype/Jacques-Francois-Shadow"
, "https://github.com/etunni/julee"
, "https://github.com/cyrealtype/Junge"
, "https://github.com/ossobuffo/jura"
, "https://github.com/solmatas/Kadwa"
, "https://github.com/vernnobile/kameronFont"
, "https://github.com/cadsondemak/kanit.git"
, "https://github.com/itfoundry/karma"
, "https://github.com/EbenSorkin/Kavoon.git"
, "https://github.com/erinmclaughlin/Khula.git"
, "https://github.com/etunni/Kite-One"
, "https://github.com/theleagueof/knewave"
, "https://github.com/etunni/kurale.git"
, "https://github.com/itfoundry/laila"
, "https://github.com/appajid/lakkireddy"
, "https://github.com/BornaIz/Lalezar"
, "https://github.com/antonxheight/Lancelot"
, "https://github.com/silnrsi/font-lateef"
, "https://github.com/googlefonts/LatoGFVersion.git"
, "https://github.com/etunni/lemon"
, "https://github.com/Gue3bara/Lemonada"
, "https://github.com/impallari/Libre-Baskerville"
, "https://github.com/impallari/Libre-Caslon-Text"
, "https://github.com/impallari/Libre-Caslon-Display"
, "https://github.com/googlefonts/LiterataFont"
, "https://github.com/impallari/The-Lobster-Font"
, "https://github.com/pravins/lohit"
, "https://github.com/cyrealtype/Lora-Cyrillic"
, "https://github.com/vernnobile/MakoFont"
, "https://github.com/vahanhovh/mardoto"
, "https://github.com/cyrealtype/Marko-One"
, "https://github.com/cyrealtype/Marmelad-Cyrillic"
, "https://github.com/typeoff/martel"
, "https://github.com/typeoff/martel_sans"
, "https://github.com/etunni/mate"
, "https://github.com/googlefonts/mavenproFont.git"
, "https://github.com/etunni/merienda"
, "https://github.com/EbenSorkin/Merriweather"
, "https://github.com/googlefonts/MetrophobicFont.git"
, "https://github.com/vernnobile/Michroma-font"
, "https://github.com/impallari/Miltonian"
, "https://github.com/JulietaUla/Montserrat"
, "https://github.com/googlefonts/MuliFont.git"
, "https://github.com/googlefonts/NunitoFont"
, "https://github.com/etunni/offside"
, "https://github.com/theleagueof/orbitron"
, "https://github.com/etunni/orienta"
, "https://github.com/googlefonts/OswaldFont"
, "https://github.com/m4rc1e/OverpassGFVersion"
, "https://github.com/KDE/oxygen-fonts"
, "https://github.com/Fonthausen/Pacifico"
, "https://github.com/VanillaandCream/Palanquin"
, "https://github.com/googlefonts/pangolin.git"
, "https://github.com/etunni/paprika"
, "https://github.com/etunni/pathway-gothic-one"
, "https://github.com/m4rc1e/PatrickHandSC"
, "https://github.com/alexeiva/philosopher"
, "https://github.com/alexeiva/play"
, "https://github.com/clauseggers/Playfair-Display"
, "https://github.com/cyrealtype/Podkova"
, "https://github.com/alexeiva/poiretone.git"
, "https://github.com/itfoundry/poppins"
, "https://github.com/Omnibus-Type/PragatiNarrow"
, "https://github.com/cyrealtype/Prata"
, "https://github.com/theleagueof/prociono"
, "https://github.com/cadsondemak/prompt"
, "https://github.com/jasperdewaard/Proza-Libre"
, "https://github.com/graphicore/Questrial-Font"
, "https://github.com/andrew-paglinawan/QuicksandFamily.git"
, "https://github.com/googlefonts/RadleyFont"
, "https://github.com/itfoundry/rajdhani"
, "https://github.com/zeynepakay/Rakkas"
, "https://github.com/impallari/Raleway.git"
, "https://github.com/antonxheight/Ranga"
, "https://github.com/christiannaths/Redacted-Font.git"
, "https://github.com/DunwichType/RhodiumLibre.git"
, "https://github.com/googlefonts/RokkittFont.git"
, "https://github.com/googlefonts/rubik.git"
, "https://github.com/alexeiva/rubikmonoone"
, "https://github.com/googlefonts/SacramentoFont"
, "https://github.com/juandelperal/sahitya"
, "https://github.com/Omnibus-Type/Sansita"
, "https://github.com/huertatipografica/sarala.git"
, "https://github.com/googlefonts/scada.git"
, "https://github.com/googlefonts/sigmarone.git"
, "https://github.com/etunni/sintony"
, "https://github.com/cyrealtype/Sirin-Stencil"
, "https://github.com/TiroTypeworks/Slabo"
, "https://github.com/theleagueof/sniglet"
, "https://github.com/theleagueof/sorts-mill-goudy"
, "https://github.com/googlefonts/SourceSerifProGFVersion.git"
, "https://github.com/googlefonts/spacemono.git"
, "https://github.com/cadsondemak/sriracha.git"
, "https://github.com/etunni/strait"
, "https://github.com/cyrealtype/Sumana.git"
, "https://github.com/huertatipografica/sura.git"
, "https://github.com/cadsondemak/taviraj"
, "https://github.com/itfoundry/teko"
, "https://github.com/appajid/timmana"
, "https://github.com/cadsondemak/trirong"
, "https://github.com/vernnobile/TrocchiFont"
, "https://github.com/etunni/unlock"
, "https://github.com/Omnibus-Type/Unna"
, "https://github.com/EbenSorkin/Vampiro"
, "https://github.com/alefalefalef/Varela-Round-Hebrew.git"
, "https://github.com/EbenSorkin/Varta.git"
, "https://github.com/motaitalic/vesper-libre"
, "https://github.com/librefonts/vibur"
, "https://github.com/cyrealtype/Vidaloka"
, "https://github.com/cyrealtype/Volkhov"
, "https://github.com/FAlthausen/Vollkorn-Typeface"
, "https://github.com/phoikoi/VT323"
, "https://github.com/etunni/warnes"
, "https://github.com/weiweihuanghuang/Work-Sans"
, "https://github.com/alexeiva/yanone-kaffeesatz"
, "https://github.com/alexeiva/yesevaone"
, "https://github.com/librefonts/zeyada"
];
main(queue.slice());
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment