Github mirror script, including organisation repos and gist mirror script.

.gitignore

config.toml
.vscode/

gistmirror.sh

#!/bin/bash
# Gist mirror script
# Jan van Haarst [email protected]
# 20150115

# Debug
#set -o xtrace
#set -o verbose

# Stop on error
set -o errexit

# Show some debug info
verbose=''

# Create a token at https://github.com/settings/tokens/new
token=''

if [ "$token" == '' ] 
then
    if [ "$GITHUB_TOKEN" != '' ]
    then
        token=$GITHUB_TOKEN
    else
        echo "Please enter a github token in the script, or run it like this:"
        echo "GITHUB_TOKEN=( token from https://github.com/settings/tokens/new ) " $(basename $0)
        echo "Optional : set the backup directory in the script, or pass BACKUP_DIR to it."
        echo "Otherwise the script will the working directory."
        exit 1
    fi
fi

# Place to store the mirrors
backup_dir=''

if [ "$backup_dir" == '' ] 
then
    if [ "$BACKUP_DIR" != '' ]
    then
        backup_dir=$BACKUP_DIR
    else
        backup_dir=$PWD
    fi
fi

# Check to see if we have jq available
if ! command -v jq > /dev/null 2>&1; then
    echo "This script needs jq (https://stedolan.github.io/jq/) to function."
    echo "Please install, and try again."
    exit 1
fi

# Get the right URL for the "gists_url"
login=$(curl -s -u ${token}:x-oauth-basic https://api.github.com/user | jq --raw-output '.login')

# Get the git_pull_urls (paginated, max 100 per page)
git_pull_url=""
page=1
while true; do
    response=$(curl -s -u "${token}:x-oauth-basic" "https://api.github.com/users/${login}/gists?per_page=100&page=${page}")
    urls=$(echo "$response" | jq -r '.[].git_pull_url')
    if [ -z "$urls" ]; then
        break
    fi
    git_pull_url="${git_pull_url} ${urls}"
    page=$((page + 1))
done

# Mirror the repo's
mkdir -p "${backup_dir}/${login}/gists"
pushd "${backup_dir}/${login}/gists" > /dev/null

for repo in $git_pull_url
do
    mirror_dir=$(basename "${repo}")
    if [ -d "${mirror_dir}" ]; then
        if [ "$verbose" != '' ];then echo "Updating " "${mirror_dir}";fi
        pushd "${mirror_dir}" > /dev/null
        git remote update
        popd > /dev/null
    else
        if [ "$verbose" != '' ];then echo "Cloning " "$repo";fi
        git clone --mirror "$repo"
    fi
done
popd > /dev/null

gitlab_branch_count.sh

#!/bin/bash
# Gitlab branch counter
# Jan van Haarst [email protected]
# 2021-07-21

# Debug
#set -o xtrace
#set -o verbose

# Stop on error
set -o errexit

# Show some debug info
verbose='1'

# Create a token at https://git.wur.nl/-/profile/personal_access_tokens
token=''

if [ "$token" == '' ] 
then
    if [ "$GITLAB_TOKEN" != '' ]
    then
        token=$GITLAB_TOKEN
    else
        echo "Please enter a gitlab token in the script, or run it like this:"
        echo "GITLAB_TOKEN=( token from https://git.wur.nl/-/profile/personal_access_tokens ) " $(basename $0)
        exit 1
    fi
fi

# Gitlab API URL
api_url="https://git.wur.nl/api/v4/"

# Check to see if we have jq available
if ! command -v jq > /dev/null 2>&1; then
    echo "This script needs jq (https://stedolan.github.io/jq/) to function."
    echo "Please install, and try again."
    exit 1
fi

# Get all the repos this token/user has acces to
list_of_repo_ids=$(curl --silent --header "PRIVATE-TOKEN: ${token}" ${api_url}'/projects?simple=false&membership=true&per_page=50000' | jq '.[].id')
for repo_id in $list_of_repo_ids
do
    name=$(curl -s --header "PRIVATE-TOKEN: ${token}" ${api_url}"/projects/${repo_id}" | jq '.path_with_namespace')
    branch_count=$(curl -s --header "PRIVATE-TOKEN: ${token}" ${api_url}"/projects/${repo_id}/repository/branches" | jq '.[].name' | wc -l)
    echo -e "$name\t$branch_count"
done

gitlabmirror.sh

#!/bin/bash
# Gitlab mirror script
# Jan van Haarst [email protected]
# 2022-10-09

# Debug
#set -o xtrace
#set -o verbose

# Stop on error
set -o errexit

# Show some debug info
verbose=''

# Get script location
# Source - https://stackoverflow.com/a/246128
# Posted by dogbane, modified by community. See post 'Timeline' for change history
# Retrieved 2026-03-09, License - CC BY-SA 4.0

SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )

# Check if we have a config file, and source it if we do. This allows us to set the token and backup directory without modifying the script.
if [ -f "${SCRIPT_DIR}"/config.toml ]
then
    # shellcheck disable=SC1091
    . "${SCRIPT_DIR}"/config.toml
fi

# Create a token at https://git.wur.nl/-/profile/personal_access_tokens
token=''

if [ "$token" == '' ] 
then
    if [ "$GITLAB_TOKEN" != '' ]
    then
        token=$GITLAB_TOKEN
    else
        echo "Please enter a gitlab token in the script, or run it like this:"
        echo "GITLAB_TOKEN=( token from https://git.wur.nl/-/profile/personal_access_tokens ) " "$(basename "$0")"
        echo "Optional : set the backup directory in the script, or pass BACKUP_DIR to it."
        echo "Otherwise the script will use the working directory."
        exit 1
    fi
fi

# Place to store the mirrors
backup_dir=''

if [ "$backup_dir" == '' ] 
then
    if [ "$BACKUP_DIR" != '' ]
    then
        backup_dir="$BACKUP_DIR"
    else
        backup_dir="$PWD"
    fi
fi
echo "Using backup directory: $backup_dir"
# Gitlab API URL
api_url="https://git.wur.nl/api/v4"


# What method do we use to login and retrieve the repos ?
method="http_url"

# Helper function to fetch all pages from a paginated GitLab API endpoint
# Usage: gitlab_fetch_all "/endpoint?param=value"
gitlab_fetch_all() {
    local endpoint="$1"
    local page=1
    local result="[]"
    while true; do
        local separator="&"
        # Use ? if the endpoint has no query parameters yet
        if [[ "$endpoint" != *"?"* ]]; then
            separator="?"
        fi
        local response
        response=$(curl --silent --header "PRIVATE-TOKEN: ${token}" "${api_url}${endpoint}${separator}per_page=100&page=${page}")
        # Stop if we get an empty array or an error
        if [ "$(echo "$response" | jq 'if type == "array" then length else 0 end')" -eq 0 ]; then
            break
        fi
        result=$(echo "$result" "$response" | jq -s '.[0] + .[1]')
        page=$((page + 1))
    done
    echo "$result"
}

# Check to see if we have jq available
if ! command -v jq > /dev/null 2>&1; then
    echo "This script needs jq (https://stedolan.github.io/jq/) to function."
    echo "Please install, and try again."
    exit 1
fi

# Get all the repos of this owner (not as part of a group)
# First get the user info:
user_id=$(curl --silent --header "PRIVATE-TOKEN: ${token}" ${api_url}"/user" | jq .id)
login=$(curl --silent --header "PRIVATE-TOKEN: ${token}" ${api_url}"/user" | jq -r .username)
list_of_repos=$(gitlab_fetch_all "/users/${user_id}/projects" | jq -r '.[].'${method}'_to_repo' | sort)

# Mirror the repo's
mkdir -p "${backup_dir}/${login}"
pushd "${backup_dir}/${login}" > /dev/null

encoded_login=$(printf %s "${login}" | jq -s -R -r @uri)
for repo in $list_of_repos
do
    # Derive the mirror directory name (e.g. "repo.git")
    mirror_dir=$(basename "${repo}")
    # Remove https://
    repo=$(echo "${repo}" | sed 's|https://||')
    # Add auth
    repo=https://${encoded_login}:${token}@${repo}

    if [ -d "${mirror_dir}" ]; then
        if [ "$verbose" != '' ];then echo "Updating " "${mirror_dir}";fi
        pushd "${mirror_dir}" > /dev/null
        git remote update
        popd > /dev/null
    else
        if [ "$verbose" != '' ];then echo "Cloning " "$repo";fi
        git clone --verbose --mirror "$repo"
    fi
done
popd > /dev/null

# ORGANIZATIONS

# Get the list of top_level groups the user is member of
group_ids=$(gitlab_fetch_all "/groups?top_level_only=true" | jq '.[].id' | sort -n)

# Loop through groups, retrieve the repos, and store them in subdirectories
for group_id in $group_ids
do
    login=$(curl --silent --header "PRIVATE-TOKEN: ${token}" ${api_url}"/groups/${group_id}" | jq -r .name)
    if [ "$verbose" != '' ];then echo "Working on $login";fi
    
    list_of_repos=''
    list_of_repos=$(gitlab_fetch_all "/groups/${group_id}/projects" | jq -r '.[].'${method}'_to_repo')
    # Mirror the repo's
    mkdir -p "${backup_dir}/${login}"
    pushd "${backup_dir}/${login}" > /dev/null

    encoded_login=$(printf %s "${login}" | jq -s -R -r @uri)
    for repo in $list_of_repos
    do
        mirror_dir=$(basename "${repo}")
        # Remove https://
        repo=$(echo "${repo}" | sed 's|https://||')
        # Add auth
        repo=https://${encoded_login}:${token}@${repo}

        if [ -d "${mirror_dir}" ]; then
            if [ "$verbose" != '' ];then echo "Updating " "${mirror_dir}";fi
            pushd "${mirror_dir}" > /dev/null
            git remote update
            popd > /dev/null
        else
            if [ "$verbose" != '' ];then echo "Cloning " "$repo";fi
            git clone --verbose --mirror "$repo"
        fi
    done
    popd > /dev/null
done

gitmirror.sh

#!/bin/bash
# Github mirror script
# Jan van Haarst [email protected]
# 20150115

# Debug
#set -o xtrace
#set -o verbose

# Stop on error
set -o errexit

# Show some debug info
verbose=''

# Create a token at https://github.com/settings/tokens/new
token=''

if [ "$token" == '' ] 
then
    if [ "$GITHUB_TOKEN" != '' ]
    then
        token=$GITHUB_TOKEN
    else
        echo "Please enter a github token in the script, or run it like this:"
        echo "GITHUB_TOKEN=( token from https://github.com/settings/tokens/new ) " $(basename $0)
        echo "Optional : set the backup directory in the script, or pass BACKUP_DIR to it."
        echo "Otherwise the script will the working directory."
        exit 1
    fi
fi

# Place to store the mirrors
backup_dir=''

if [ "$backup_dir" == '' ] 
then
    if [ "$BACKUP_DIR" != '' ]
    then
        backup_dir="$BACKUP_DIR"
    else
        backup_dir="$PWD"
    fi
fi

# Github API URL
api_url="https://api.github.com"

# What method do we use to login and retrieve the repos ?
method="ssh_url"

# Check to see if we have jq available
if ! command -v jq > /dev/null 2>&1; then
    echo "This script needs jq (https://stedolan.github.io/jq/) to function."
    echo "Please install, and try again."
    exit 1
fi

# Get the right URL for the "current_user_url"
current_user_url=$(curl -s -u ${token}:x-oauth-basic https://api.github.com | jq --raw-output '.current_user_url')

# Get the right URL for the "organization_repositories_url"
organization_repositories_url=$(curl -s -u ${token}:x-oauth-basic https://api.github.com | jq --raw-output '.organization_repositories_url')

# What is the login that belongs to this token ?
login=$(curl -s -u ${token}:x-oauth-basic $current_user_url | jq --raw-output '.login')

# Get the repos_url and organizations_url of this user
repos_url=$(curl -s -u ${token}:x-oauth-basic $current_user_url | jq --raw-output '.repos_url')

# Get the user repo's (paginated, max 100 per page)
list_of_repos=''
page=1
while true; do
    repos=$(curl -s -u "${token}:x-oauth-basic" "${repos_url}?page=${page}&per_page=100" | jq --raw-output '.[].'${method}'')
    if [ -z "$repos" ]; then
        break
    fi
    list_of_repos="$list_of_repos $repos"
    page=$((page + 1))
done

# Mirror the repo's
mkdir -p "${backup_dir}/${login}"
pushd "${backup_dir}/${login}" > /dev/null

for repo in $list_of_repos
do
    mirror_dir=$(basename "${repo}")
    if [ -d "${mirror_dir}" ]; then
        if [ "$verbose" != '' ];then echo "Updating " "${mirror_dir}";fi
        pushd "${mirror_dir}" > /dev/null
        git remote update
        popd > /dev/null
    else
        if [ "$verbose" != '' ];then echo "Cloning " "$repo";fi
        git clone --mirror "$repo"
    fi
done
popd > /dev/null

# ORGANIZATIONS

# Get the list of repos_url for the organizations the user is member of
ORGS_logins=$(curl -s -u ${token}:x-oauth-basic https://api.github.com/user/orgs | jq --raw-output '.[].login' )

# Loop through organizations, retrieve the repos, and store them in subdirectories
for login in $ORGS_logins
do
    if [ "$verbose" != '' ];then echo echo "Working on $login";fi
    organization_repos_url=$(curl -s -u ${token}:x-oauth-basic https://api.github.com/orgs/${login} |  jq --raw-output '.repos_url' )
    list_of_repos=''
    page=1
    while true; do
        repos=$(curl -s -u "${token}:x-oauth-basic" "${organization_repos_url}?page=${page}&per_page=100" | jq --raw-output '.[].'${method}'')
        if [ -z "$repos" ]; then
            break
        fi
        list_of_repos="$list_of_repos $repos"
        page=$((page + 1))
    done
    mkdir -p "${backup_dir}/${login}"
    pushd "${backup_dir}/${login}" > /dev/null
    for repo in $list_of_repos
    do
        mirror_dir=$(basename "${repo}")
        if [ -d "${mirror_dir}" ]; then
            if [ "$verbose" != '' ];then echo "Updating " "${mirror_dir}";fi
            pushd "${mirror_dir}" > /dev/null
            git remote update
            popd > /dev/null
        else
            if [ "$verbose" != '' ];then echo "Cloning " "$repo";fi
            git clone --mirror "$repo"
        fi
    done
    popd > /dev/null
done