Mirror GitHub repos locally on NAS

mirrorGitHub.sh

#!/bin/bash
# Mirrors all GitHub repositories the current user has read access to.
# See license at the end of the file.

# Token from https://github.com/settings/tokens
OAUTH_TOKEN="<TODO TOKEN>"

# where should the files be saved
DIR="<TODO PATH>"

# Creates the following directory structure:
# DIR
# ├── user1
# │   ├── repo1
# │   └── repo2
# ├── org1
# │   ├── repo3
# │   └── repo4


# Based on the following scripts
#   - https://jojozhuang.github.io/tutorial/backup-github-repositories-to-synology-nas/ by Rong Zhuang
#   - https://gist.github.com/infertux/950441 by Cédric Félizard
#   - https://gist.github.com/mbohun/b161521b2440b9f08b59 by Martin Bohun Hormann


# === SCRIPT START ===

# GitHub API URL
githubReposUrl="https://api.github.com/user/repos?per_page=100"

# Count the of repositories
fetchCount=0
cloneCount=0

updateOrCloneRepo () {
  owner=$1
  name=$2
  cloneUrl=$3

  cloneUrlWithToken="${cloneUrl/\/\////$OAUTH_TOKEN@}"
  ownerDir="${DIR}/${owner}"
  repoDir="${ownerDir}/${name}.git"
  
  if [ -d "$repoDir" ] ; then
      let fetchCount++
      echo "Fetching ${owner}/${name}"
      cd "$repoDir" || exit
      git remote set-url origin "$cloneUrlWithToken" # To use the current token
      git fetch
  else
      let cloneCount++
      mkdir "$ownerDir" -p
      cd "$ownerDir" || exit
      echo "Cloning ${owner}/${name}"
      git clone --mirror "$cloneUrlWithToken"
  fi
}
  
updateOrCloneAllRepos() {
    echo "Fetching list of all repositories"
    pages=$(curl -H "Authorization: token ${OAUTH_TOKEN}" -s -I "${githubReposUrl}" | grep '^link:' | sed -e 's/^link:.*page=//g' -e 's/>.*$//g')
    
    repos=""
    for p in $(seq 1 "$pages"); do
      echo "${githubReposUrl}&page=${p}"
      repos="${repos:+$repos' '}" # add a space if repos is not empty
      repos+=$(curl -H "Authorization: token ${OAUTH_TOKEN}" -s "${githubReposUrl}&page=${p}" | jq -r '.[] | "\(.owner.login),\(.name),\(.clone_url)"')
    done
    
    for repo in $repos
    do
      owner=$(echo "$repo" | cut -d ',' -f1)
      name=$(echo "$repo" | cut -d ',' -f2)
      cloneUrl=$(echo "$repo" | cut -d ',' -f3)

      updateOrCloneRepo "$owner" "$name" "$cloneUrl"
    done
}
 
updateOrCloneAllRepos

echo "${fetchCount} repositories updated"
echo "${cloneCount} new repositories cloned"

exit 0

# Copyright 2022 Robin Pokorny <[email protected]>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.