Skip to content

Instantly share code, notes, and snippets.

@radekg
Created December 9, 2012 06:34
Show Gist options
  • Save radekg/4243613 to your computer and use it in GitHub Desktop.
Save radekg/4243613 to your computer and use it in GitHub Desktop.
Backup all github repos from all organisations we are assigned to and put on S3 (node.js)
var https = require("https")
, fs = require("fs")
, spawn = require('child_process').spawn
, AWS = require('aws-sdk');
var CONFIG = {
username: "<github-username>"
, password: "<github-password>"
, backupdir: "/where/to"
, bucket: "github.backup"
};
AWS.config.update({ accessKeyId: "<ACCESS_KEY>", secretAccessKey: "<SECRET>" });
s3 = new AWS.S3();
var state = {
numOrgs: 0,
fetchedOrgs: 0,
totalRepos: 0,
processedRepos: 0,
repos: {}
};
function githubAccessToken(username, password, callback) {
var postData = "{\"scopes\":[\"repo\"]}";
var responseData = "";
var githubRequest = https.request({
hostname: "api.github.com"
, port: 443
, path: "/authorizations"
, method: "POST"
, headers: {
'Content-Length': postData.length
, 'Authorization': "Basic " + new Buffer(username + ':' + password).toString('base64')
}
}, function(response) {
response.on("data", function(data) {
responseData += data+"";
});
response.on("end", function() {
var token = JSON.parse(responseData).token;
githubUserOrganisations( username, token, function(err, orgs) {
state.numOrgs = orgs.length;
orgs.forEach(function(org) {
githubOrganizationRepos( org.login, token );
});
});
});
});
githubRequest.end(postData);
}
function githubUserOrganisations(username, accessToken, callback) {
var fullData = "";
var githubRequest = https.request({
hostname: "api.github.com"
, port: 443
, path: "/users/" + username + "/orgs?access_token=" + accessToken
, method: "GET"
}, function(response) {
response.on("data", function(data) {
fullData += data;
});
response.on("end", function() {
callback(null, JSON.parse(fullData));
});
response.on("error", function(e) {
callback("Error while requesting user organisations: " + e, null);
});
});
githubRequest.end();
}
function githubOrganizationRepos(orgName, accessToken) {
var fullData = "";
var githubRequest = https.request({
hostname: "api.github.com"
, port: 443
, path: "/orgs/" + orgName + "/repos?access_token=" + accessToken
, method: "GET"
}, function(response) {
response.on("data", function(data) {
fullData += data;
});
response.on("end", function() {
state.fetchedOrgs++;
state.repos[ orgName ] = JSON.parse( fullData );
state.totalRepos += state.repos[ orgName ].length;
if ( state.numOrgs == state.fetchedOrgs ) {
for ( var key in state.repos ) {
state.repos[ orgName ].forEach(function(repo) {
backupRepo( key, repo.name, repo.ssh_url );
});
}
}
});
response.on("error", function(e) {
callback("Error while requesting organization repos: " + e, null);
});
});
githubRequest.end();
}
function backupRepo( orgName, repoName, sshUrl ) {
if ( fs.existsSync( CONFIG.backupdir + "/github.backup" ) === false ) {
fs.mkdirSync( CONFIG.backupdir + "/github.backup" );
}
if ( fs.existsSync( CONFIG.backupdir + "/github.backup/" + orgName ) === false ) {
fs.mkdirSync( CONFIG.backupdir + "/github.backup/" + orgName );
}
if ( fs.existsSync( CONFIG.backupdir + "/github.backup/" + orgName + "/" + repoName ) === false ) {
var gitCmd = spawn("git", ["clone", sshUrl, CONFIG.backupdir + "/github.backup/" + orgName + "/" + repoName ]);
gitCmd.on("exit", function() {
console.log("Repository " + orgName + "/" + repoName + " successfully cloned. Zipping...");
putOnS3( orgName, repoName );
});
} else {
var gitCmd = spawn("git", ["pull", "--all", "--git-dir", CONFIG.backupdir + "/github.backup/" + orgName + "/" + repoName + "/.git" ]);
gitCmd.on("exit", function() {
console.log("Repository " + orgName + "/" + repoName + " successfully pulled. Zipping...");
putOnS3( orgName, repoName );
});
}
}
function putOnS3( orgName, repoName ) {
var zipCmd = spawn("zip", ["-r", CONFIG.backupdir + "/github.backup/" + orgName + "/" + repoName + ".zip", CONFIG.backupdir + "/github.backup/" + orgName + "/" + repoName]);
zipCmd.on("exit", function() {
console.log(CONFIG.backupdir + "/github.backup/" + orgName + "/" + repoName + ".zip created. Attempting S3 upload.");
fs.readFile(CONFIG.backupdir + "/github.backup/" + orgName + "/" + repoName + ".zip", function(err, data) {
s3.client.putObject({ Bucket: CONFIG.bucket, Key: orgName+"/"+repoName+".zip", Body: data }).done(function(resp) {
console.log(CONFIG.backupdir + "/github.backup/" + orgName + "/" + repoName + ".zip uploaded. Cleaning up...");
fs.unlinkSync(CONFIG.backupdir + "/github.backup/" + orgName + "/" + repoName + ".zip");
console.log(CONFIG.backupdir + "/github.backup/" + orgName + "/" + repoName + ".zip removed after upload.");
state.processedRepos++;
if ( state.totalRepos === state.processedRepos ) {
process.exit(0);
}
});
});
});
}
githubAccessToken(CONFIG.username, CONFIG.password);
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment