A Node.js script that clones all non-forked repos and all gists on GitHub for a user.

ghclone.js

/*
 * A Node.js script that clones all non-forked repos and all gists on GitHub
 * for a user. If the repos/gists have been cloned already, the script pulls
 * new commits from GitHub instead.
 *
 * Usage:
 *
 *   node ghclone.js -u <user> -p <password> -d <destination dir>
 *
 * Tested with Node.js version 0.6.7.
 *
 * Author: Per Rovegard <[email protected]>
 */

var https = require('https'),
    exec = require('child_process').exec,
    path = require('path'),
    util = require('util'),
    events = require('events'),
    emitter = new events.EventEmitter();

// Configure the emitter to print messages to the console, and to exit if we
// run into a severe condition.
emitter
  .on('info', console.info)
  .on('error', console.error)
  .on('severe', function(err) {
    console.error(err);
    process.exit(1);
  });

// Used by exec to print the result of executing a subprocess.
function puts(error, stdout, stderr) {
  if (error) {
    emitter.emit('error', error);
  } else {
    emitter.emit('info', stdout.trim());
    if (stderr) {
      emitter.emit('error', stderr.trim());
    }
  }
}

// Assembles the value of the Authorization header passed in the HTTPS request
// to GitHub.
function auth(user, pass) {
  return 'Basic ' + (new Buffer(user + ':' + pass)).toString('base64'); 
}

// Performs a git pull in the given git directory.
function git_pull(gitdir) {
  exec(util.format('git --git-dir=%s/.git pull', gitdir), puts);
}

// Performs a git clone of the given URL into the given directory.
function git_clone(url, dir) {
  exec(util.format('git clone %s %s', url, dir), puts);
}

// Create a worker for do_clone to be used in the call to path.exists.
// Required to get the loop variable values correct.
function create_worker(gitdir, clone_url) {
  return function(exists) {
    if (exists) {
      emitter.emit('info', util.format('%s exists, will pull.', gitdir));
      git_pull(gitdir);
    } else {
      emitter.emit('info', util.format('%s does not exist, will clone from %s.', gitdir, clone_url));
      git_clone(clone_url, gitdir);
    }
  }
}

/* Performs the actual process of iterating over repositories and gists, and
 * cloning or pulling depending on whether or not the corresponding
 * destination directory already exists. Forked repositories are excluded
 * during iteration, and will thus not be cloned.
 *
 * The first parameter takes the JSON data coming from the server, and the
 * second takes the destination directory into which individual repositories
 * and gists are cloned (i.e., they are cloned into subdirectories of this
 * directory). The directory is assumed to exist at this point.
 */
function do_clone(data, destdir) {
  var json = JSON.parse(data);
  if (json.message) {
    emitter.emit('severe', util.format('GitHub responded: %s', json.message));
  }
  json.forEach(function(repo) {
    if (!repo.fork) {
      var name = repo.name || repo.id;
      var clone_url = repo.clone_url || repo.git_pull_url;
      var gitdir = path.join(destdir, name);
      path.exists(gitdir, create_worker(gitdir, clone_url));
    }
  });
}

// Main entry point of the program.
function main(username, password, destdir) {
  var http_options = {
    host: 'api.github.com',
    headers: {
      'Authorization': auth(username, password)
    }
  };

  ['/user/repos', '/gists'].forEach(function(path) {
    http_options.path = path;
    https.get(http_options, function(res) {
      var data = '';
      res.on('data', function(d) {
        data += d;
      });
      res.on('end', function() {
        do_clone(data, destdir);
      });
    }).on('error', function(e) { 
      emitter.emit('error', e);
    });
  });
}

/* Simple argument parser that looks for arguments that begin with a dash, 
 * and requires each such argument to have a non-option associated value.
 * Options and their values are stored in an associative array which is the
 * result of this function.
 */
function parse_args(args) {
  var opts = {};
  for (var i = 0; i < args.length; i++) {
    if (args[i][0] == '-') {
      var value = args[i + 1];
      if (value && value[0] != '-') {
        eval(util.format('opts.%s = "%s"', args[i].substring(1), value));
        i++;
      }
    }
  }
  return opts;
}

var opts = parse_args(process.argv.slice(2)),
    user = opts.u,
    password = opts.p,
    destdir = opts.d;

// Sanity check of arguments.
if (!(user && password && destdir)) {
  emitter.emit('severe', util.format('Usage: node %s -u <username> -p <password> -d <destination dir>', process.argv[1]));
}

path.exists(destdir, function(exists) {
  if (exists) {
    // Run the show!
    main(user, password, destdir);
  } else {
    // The destination directory must exist!
    emitter.emit('severe', util.format('Destination directory %s cannot be found.', destdir));
  }
});