backup-github-org.sh

#!/bin/bash 
#
# SOURCE: this is an adaptation of the following gists:
#   * https://gist.github.com/rodw/3073987 - original
#   * https://gist.github.com/bjtitus/8851816 - modification of the original /w pagination for more than 30 repos
#
# SCOPE: backup an organization's GitHub repositories with issues and wikis
#
# Uses the GitHub API to pull down the list of repos & clone them
#
# Fast setup: change GITHUB_ORG & GITHUB_TOKEN below
#

# Exit on error
set -e
# Treat unset variables as an error
set -u

BACKUP_DIR="/backups/github"                                                                   # where to place the backup files on the local disk; absolute path
GITHUB_ORG="<CHANGE-ME>"                                                                       # the GitHub organization whose repos will be backed up
PRUNE_OLD="true"                                                                               # when `true`, old backups will be deleted
PRUNE_AFTER="30"                                                                               # the min age (in days) of backup files to delete
SILENT="true"                                                                                  # when `true`, only show error messages 
GITHUB_API="https://api.github.com"                                                            # base URI for the GitHub API
GITHUB_TOKEN="<CHANGE-ME>"                                                                     # the token to use for GitHub API Authentication
GITHUB_GIT_CLONE_CMD="/usr/bin/git clone --quiet --mirror https://${GITHUB_TOKEN}@github.com/" # base command for cloning GitHub repos
TSTAMP=$(/bin/date "+%Y-%m-%dT%H:%M:%S-%Z")                                                    # timestamp
ECHO="/bin/echo"                                                                               # binaries, binaries, binaries
CURL="/usr/bin/curl -s"                                                                        # make curl silent by default
NEXT_PAGE="${GITHUB_API}/orgs/${GITHUB_ORG}/repos?access_token=${GITHUB_TOKEN}"                # URL for the next page
REPOLIST=""                                                                                    # initializing...

# The function `check` will notify if the given command fails; not exiting, we still need backups
function check() {
    "$@"
    STATUS=$?
    if [ ${STATUS} -ne 0 ]; then
        ${ECHO} "ERROR: Encountered error (${STATUS}) while running the following:" >&2
        ${ECHO} "           $@" >&2
        ${ECHO} "       (at line ${BASH_LINENO[0]} of file $0.)" >&2
    fi
}

# The function `tgzrm` will create a gzipped tar archive of the specified file ($1) and then remove the original
function tgzrm {
    check /usr/bin/tar -czf $1.tar.gz -C / $(${ECHO} "$1" | check /usr/bin/sed 's/^\///') && check /bin/rm -rf $1
}

${SILENT} || ${ECHO} "${TSTAMP} === INITIALIZING ==="

${SILENT} || ${ECHO} "${TSTAMP} Using backup directory ${BACKUP_DIR}"
if [ ! -d ${BACKUP_DIR} ]; then
    check /bin/mkdir -p ${BACKUP_DIR}
fi

${SILENT} || echo "${TSTAMP} Fetching list of repositories for ${GITHUB_ORG}... "
while [ ${NEXT_PAGE} ]; do
    GET_URL_CONTENT="$(check ${CURL} -i ${NEXT_PAGE} -q)"
    REPOLIST=${REPOLIST}$(check ${ECHO} "${GET_URL_CONTENT}" | check /usr/bin/awk -F': "' '/"name"/ {print $2}' | check /usr/bin/sed -e 's/",//g')$'\n'
    NEXT_PAGE=$(check ${ECHO} "${GET_URL_CONTENT}" | check /usr/bin/awk -F'<' '/"next"/ {print $2}' | check /usr/bin/sed -e 's/>;.*//g')
done
${SILENT} || ${ECHO} "${TSTAMP} Found $(${ECHO} ${REPOLIST} | /usr/bin/wc -w | /usr/bin/sed 's/ //g') repositories."

${SILENT} || ${ECHO} "${TSTAMP} === BACKING UP ==="
for REPO in ${REPOLIST}; do
    ${SILENT} || ${ECHO} "${TSTAMP} Backing up ${GITHUB_ORG}/${REPO}"
    check ${GITHUB_GIT_CLONE_CMD}${GITHUB_ORG}/${REPO}.git ${BACKUP_DIR}/${REPO}-${TSTAMP}.git && tgzrm ${BACKUP_DIR}/${REPO}-${TSTAMP}.git

    ${SILENT} || ${ECHO} "${TSTAMP} Backing up ${GITHUB_ORG}/${REPO}.wiki"
    check ${GITHUB_GIT_CLONE_CMD}${GITHUB_ORG}/${REPO}.wiki.git ${BACKUP_DIR}/${REPO}.wiki-${TSTAMP}.git && tgzrm ${BACKUP_DIR}/${REPO}.wiki-${TSTAMP}.git

    ${SILENT} || ${ECHO} "${TSTAMP} Backing up ${GITHUB_ORG}/${REPO} issues"
    check ${CURL} ${GITHUB_API}/repos/${GITHUB_ORG}/${REPO}/issues?access_token=${GITHUB_TOKEN} -q > ${BACKUP_DIR}/${REPO}.issues-${TSTAMP} && tgzrm ${BACKUP_DIR}/${REPO}.issues-${TSTAMP}
done

if [ ${PRUNE_OLD} ]; then
    ${SILENT} || ${ECHO} "${TSTAMP} === PRUNING ==="
    ${SILENT} || ${ECHO} "${TSTAMP} Pruning backup files ${PRUNE_AFTER} days old or older."
    ${SILENT} || ${ECHO} "${TSTAMP} Found $(/usr/bin/find ${BACKUP_DIR} -name '*.tar.gz' -mtime +${PRUNE_AFTER} | /usr/bin/wc -l | /usr/bin/sed 's/ //g') files to prune."
    check /usr/bin/find ${BACKUP_DIR} -name '*.tar.gz' -mtime +${PRUNE_AFTER} -exec /bin/rm -fv {} \; 
fi

${SILENT} || ${ECHO} "${TSTAMP} GitHub backup completed."
${SILENT} || ${ECHO} "${TSTAMP} === DONE ===" && ${ECHO} ""