script to clone a list of gists, only those with markdown files and a package.json

gists-clone.js

// script to clone a list of gists, only those with markdown files

var child = require('child_process'),
  colors = require('colors'),
  path = require('path'),
  fs = require('fs');

// set up your credentials
var cred = "user:PASSWORD";

// set the relative output dir
var output = 'gists/';

var spawn = function spawn(cmd, args, callback) {
  var stderr = [], stdout = [],
      ch = child.spawn(cmd, args);

  ch.stdout.pipe(process.stdout, {end: false});
  ch.stderr.pipe(process.stderr);
  ch.stdout.on('data', function(data) { stdout[stdout.length] = data; });
  ch.stderr.on('data', function(data) { stderr[stderr.length] = data; });
  ch.on('exit', function (code) {
    stdout = stdout.join('\n');
    stderr = stderr.join('\n');
    if(callback) callback(code, stdout, stderr);
    ch.emit('done', code, stdout, stderr);
  });

  return ch;
};

var error = function error() {
  var args = Array.prototype.slice.call(arguments);
  console.error(args.join(' ').red.bold);
  process.exit(1);
};


console.log("Cool. Let's start".grey.bold);


// deal with pagination
//

(function curl(page) {

  page = page || 1;
  console.log(('Fetching gists... page ' + page).grey);
  child.exec(['curl', '-u', cred, 'https://api.github.com/gists?page=' + page].join(' '), function(code, stdout, stderr) {
    if(code) return error('curl exited with code ' + code);

    var data = JSON.parse(stdout);
    if(data.message) return error(data.message);

    console.log('✔ Github responded with some files...'.green.bold);

    if(!data.length) return console.log("\n\n✔ We're done");

    var gists = data.filter(function(gist) {
      var keys = Object.keys(gist.files),
      pkg = !!~keys.indexOf('package.json'),
      posts = keys.filter(function(file) {
        return /\.md$/.test(file);
      });

      return pkg && posts.length; 
    });

    if(gists.length) console.log(('  Going to clone ' + gists.length + ' gists...').green);

    (function clone(gist) {
      if(!gist) return curl(page + 1);
      var dir = path.join(output, gist.id);

      path.exists(dir, function(exists) {
        console.log([(exists ? 'Pulling ' : 'Cloning '), gist.git_push_url, ' into ' + dir].join(' ').grey.bold);

        var commands = exists ? 
          ['--git-dir', path.join(dir, '.git'), 'pull'] :
          ['clone', gist.git_push_url, dir];

        spawn('git', commands, function (code, out, err) {
          if(code) return error('git exited with code ' + code + '\n' + out + '\n\n\n' + err);
          clone(gists.pop());
        });
      });

       
    })(gists.pop());
  });

})();