Created
December 9, 2012 06:34
-
-
Save radekg/4243613 to your computer and use it in GitHub Desktop.
Backup all github repos from all organisations we are assigned to and put on S3 (node.js)
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
var https = require("https") | |
, fs = require("fs") | |
, spawn = require('child_process').spawn | |
, AWS = require('aws-sdk'); | |
var CONFIG = { | |
username: "<github-username>" | |
, password: "<github-password>" | |
, backupdir: "/where/to" | |
, bucket: "github.backup" | |
}; | |
AWS.config.update({ accessKeyId: "<ACCESS_KEY>", secretAccessKey: "<SECRET>" }); | |
s3 = new AWS.S3(); | |
var state = { | |
numOrgs: 0, | |
fetchedOrgs: 0, | |
totalRepos: 0, | |
processedRepos: 0, | |
repos: {} | |
}; | |
function githubAccessToken(username, password, callback) { | |
var postData = "{\"scopes\":[\"repo\"]}"; | |
var responseData = ""; | |
var githubRequest = https.request({ | |
hostname: "api.github.com" | |
, port: 443 | |
, path: "/authorizations" | |
, method: "POST" | |
, headers: { | |
'Content-Length': postData.length | |
, 'Authorization': "Basic " + new Buffer(username + ':' + password).toString('base64') | |
} | |
}, function(response) { | |
response.on("data", function(data) { | |
responseData += data+""; | |
}); | |
response.on("end", function() { | |
var token = JSON.parse(responseData).token; | |
githubUserOrganisations( username, token, function(err, orgs) { | |
state.numOrgs = orgs.length; | |
orgs.forEach(function(org) { | |
githubOrganizationRepos( org.login, token ); | |
}); | |
}); | |
}); | |
}); | |
githubRequest.end(postData); | |
} | |
function githubUserOrganisations(username, accessToken, callback) { | |
var fullData = ""; | |
var githubRequest = https.request({ | |
hostname: "api.github.com" | |
, port: 443 | |
, path: "/users/" + username + "/orgs?access_token=" + accessToken | |
, method: "GET" | |
}, function(response) { | |
response.on("data", function(data) { | |
fullData += data; | |
}); | |
response.on("end", function() { | |
callback(null, JSON.parse(fullData)); | |
}); | |
response.on("error", function(e) { | |
callback("Error while requesting user organisations: " + e, null); | |
}); | |
}); | |
githubRequest.end(); | |
} | |
function githubOrganizationRepos(orgName, accessToken) { | |
var fullData = ""; | |
var githubRequest = https.request({ | |
hostname: "api.github.com" | |
, port: 443 | |
, path: "/orgs/" + orgName + "/repos?access_token=" + accessToken | |
, method: "GET" | |
}, function(response) { | |
response.on("data", function(data) { | |
fullData += data; | |
}); | |
response.on("end", function() { | |
state.fetchedOrgs++; | |
state.repos[ orgName ] = JSON.parse( fullData ); | |
state.totalRepos += state.repos[ orgName ].length; | |
if ( state.numOrgs == state.fetchedOrgs ) { | |
for ( var key in state.repos ) { | |
state.repos[ orgName ].forEach(function(repo) { | |
backupRepo( key, repo.name, repo.ssh_url ); | |
}); | |
} | |
} | |
}); | |
response.on("error", function(e) { | |
callback("Error while requesting organization repos: " + e, null); | |
}); | |
}); | |
githubRequest.end(); | |
} | |
function backupRepo( orgName, repoName, sshUrl ) { | |
if ( fs.existsSync( CONFIG.backupdir + "/github.backup" ) === false ) { | |
fs.mkdirSync( CONFIG.backupdir + "/github.backup" ); | |
} | |
if ( fs.existsSync( CONFIG.backupdir + "/github.backup/" + orgName ) === false ) { | |
fs.mkdirSync( CONFIG.backupdir + "/github.backup/" + orgName ); | |
} | |
if ( fs.existsSync( CONFIG.backupdir + "/github.backup/" + orgName + "/" + repoName ) === false ) { | |
var gitCmd = spawn("git", ["clone", sshUrl, CONFIG.backupdir + "/github.backup/" + orgName + "/" + repoName ]); | |
gitCmd.on("exit", function() { | |
console.log("Repository " + orgName + "/" + repoName + " successfully cloned. Zipping..."); | |
putOnS3( orgName, repoName ); | |
}); | |
} else { | |
var gitCmd = spawn("git", ["pull", "--all", "--git-dir", CONFIG.backupdir + "/github.backup/" + orgName + "/" + repoName + "/.git" ]); | |
gitCmd.on("exit", function() { | |
console.log("Repository " + orgName + "/" + repoName + " successfully pulled. Zipping..."); | |
putOnS3( orgName, repoName ); | |
}); | |
} | |
} | |
function putOnS3( orgName, repoName ) { | |
var zipCmd = spawn("zip", ["-r", CONFIG.backupdir + "/github.backup/" + orgName + "/" + repoName + ".zip", CONFIG.backupdir + "/github.backup/" + orgName + "/" + repoName]); | |
zipCmd.on("exit", function() { | |
console.log(CONFIG.backupdir + "/github.backup/" + orgName + "/" + repoName + ".zip created. Attempting S3 upload."); | |
fs.readFile(CONFIG.backupdir + "/github.backup/" + orgName + "/" + repoName + ".zip", function(err, data) { | |
s3.client.putObject({ Bucket: CONFIG.bucket, Key: orgName+"/"+repoName+".zip", Body: data }).done(function(resp) { | |
console.log(CONFIG.backupdir + "/github.backup/" + orgName + "/" + repoName + ".zip uploaded. Cleaning up..."); | |
fs.unlinkSync(CONFIG.backupdir + "/github.backup/" + orgName + "/" + repoName + ".zip"); | |
console.log(CONFIG.backupdir + "/github.backup/" + orgName + "/" + repoName + ".zip removed after upload."); | |
state.processedRepos++; | |
if ( state.totalRepos === state.processedRepos ) { | |
process.exit(0); | |
} | |
}); | |
}); | |
}); | |
} | |
githubAccessToken(CONFIG.username, CONFIG.password); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment