Github mirror script, including organisation repos and gist mirror script.

gistmirror.sh

#!/bin/bash
# Gist mirror script
# Jan van Haarst [email protected]
# 20150115

# Debug
#set -o xtrace
#set -o verbose

# Stop on error
set -o errexit

# Show some debug info
verbose=''

# Create a token at https://github.com/settings/tokens/new
token=''

if [ "$token" == '' ] 
then
    if [ "$GITHUB_TOKEN" != '' ]
    then
        token=$GITHUB_TOKEN
    else
        echo "Please enter a github token in the script, or run it like this:"
        echo "GITHUB_TOKEN=( token from https://github.com/settings/tokens/new ) " $(basename $0)
        echo "Optional : set the backup directory in the script, or pass BACKUP_DIR to it."
        echo "Otherwise the script will the working directory."
        exit 1
    fi
fi

# Place to store the mirrors
backup_dir=''

if [ "$backup_dir" == '' ] 
then
    if [ "$BACKUP_DIR" != '' ]
    then
        backup_dir=$BACKUP_DIR
    else
        backup_dir=$PWD
    fi
fi

# Check to see if we have jq available
if ! command -v jq > /dev/null 2>&1; then
    echo "This script needs jq (https://stedolan.github.io/jq/) to function."
    echo "Please install, and try again."
    exit 1
fi

# Get the right URL for the "gists_url"
login=$(curl -s -u ${token}:x-oauth-basic https://api.github.com/user | jq --raw-output '.login')

# Get the git_pull_urls
git_pull_url=$(curl -s -u ${token}:x-oauth-basic https://api.github.com/users/${login}/gists | jq -r '.[].git_pull_url')

# Mirror the repo's
mkdir -p "${backup_dir}/${login}/gists"
pushd "${backup_dir}/${login}/gists" > /dev/null

for repo in $git_pull_url
do
    # We need to allow errors here, as git will give an error if the repo was already present
    set +o errexit
    if [ "$verbose" != '' ];then echo "Cloning " $repo;fi
    git clone --mirror "$repo"
    # Stop on error
    set -o errexit
done

# Now we can do an update on the ones we already have
for dir in `find . -maxdepth 1 -type d -name '*.git'`
do
    pushd "$dir" > /dev/null
    if [ "$verbose" != '' ];then echo "Updating " $dir;fi
    git remote update
    popd > /dev/null
done
popd > /dev/null

gitlab_branch_count.sh

#!/bin/bash
# Gitlab branch counter
# Jan van Haarst [email protected]
# 2021-07-21

# Debug
#set -o xtrace
#set -o verbose

# Stop on error
set -o errexit

# Show some debug info
verbose='1'

# Create a token at https://git.wur.nl/-/profile/personal_access_tokens
token=''

if [ "$token" == '' ] 
then
    if [ "$GITLAB_TOKEN" != '' ]
    then
        token=$GITLAB_TOKEN
    else
        echo "Please enter a gitlab token in the script, or run it like this:"
        echo "GITLAB_TOKEN=( token from https://git.wur.nl/-/profile/personal_access_tokens ) " $(basename $0)
        exit 1
    fi
fi

# Gitlab API URL
api_url="https://git.wur.nl/api/v4/"

# Check to see if we have jq available
if ! command -v jq > /dev/null 2>&1; then
    echo "This script needs jq (https://stedolan.github.io/jq/) to function."
    echo "Please install, and try again."
    exit 1
fi

# Get all the repos this token/user has acces to
list_of_repo_ids=$(curl --silent --header "PRIVATE-TOKEN: ${token}" ${api_url}'/projects?simple=false&membership=true&per_page=50000' | jq '.[].id')
for repo_id in $list_of_repo_ids
do
    name=$(curl -s --header "PRIVATE-TOKEN: ${token}" ${api_url}"/projects/${repo_id}" | jq '.path_with_namespace')
    branch_count=$(curl -s --header "PRIVATE-TOKEN: ${token}" ${api_url}"/projects/${repo_id}/repository/branches" | jq '.[].name' | wc -l)
    echo -e "$name\t$branch_count"
done

gitlabmirror.sh

#!/bin/bash
# Gitlab mirror script
# Jan van Haarst [email protected]
# 2022-10-09

# Debug
#set -o xtrace
#set -o verbose

# Stop on error
set -o errexit

# Show some debug info
verbose=''

# Create a token at https://git.wur.nl/-/profile/personal_access_tokens
token=''

if [ "$token" == '' ] 
then
    if [ "$GITLAB_TOKEN" != '' ]
    then
        token=$GITLAB_TOKEN
    else
        echo "Please enter a gitlab token in the script, or run it like this:"
        echo "GITLAB_TOKEN=( token from https://git.wur.nl/-/profile/personal_access_tokens ) " $(basename $0)
        echo "Optional : set the backup directory in the script, or pass BACKUP_DIR to it."
        echo "Otherwise the script will use the working directory."
        exit 1
    fi
fi

# Place to store the mirrors
backup_dir=''

if [ "$backup_dir" == '' ] 
then
    if [ "$BACKUP_DIR" != '' ]
    then
        backup_dir="$BACKUP_DIR"
    else
        backup_dir="$PWD"
    fi
fi

# Gitlab API URL
api_url="https://git.wur.nl/api/v4/"


# What method do we use to login and retrieve the repos ?
method="http_url"

# Check to see if we have jq available
if ! command -v jq > /dev/null 2>&1; then
    echo "This script needs jq (https://stedolan.github.io/jq/) to function."
    echo "Please install, and try again."
    exit 1
fi

# Get all the repos of this owner (not as part of a group)
# First get the user info:
user_id=$(curl --silent --header "PRIVATE-TOKEN: ${token}" ${api_url}"/user" | jq .id)
login=$(curl --silent --header "PRIVATE-TOKEN: ${token}" ${api_url}"/user" | jq -r .username)
# We need to allow errors here, as the jq will return an errorcode if no match is found
set +o errexit
list_of_repos=$(curl --silent --header "PRIVATE-TOKEN: ${token}" ${api_url}"/users/${user_id}/projects?simple=true&per_page=50000" | jq -r '.[].'${method}'_to_repo' | sort)
# Stop on error
set -o errexit

# Mirror the repo's
mkdir -p "${backup_dir}/${login}"
pushd "${backup_dir}/${login}" > /dev/null

for repo in $list_of_repos
do
    # We need to allow errors here, as git will give an error if the repo was already present
    set +o errexit
    if [ "$verbose" != '' ];then echo "Cloning " $repo;fi
    # Remove https://
    repo=$(echo ${repo} | sed 's|https://||')
    # encode login
    encoded_login=$(printf %s "${login}" | jq -s -R -r @uri)
    # Add auth
    repo=https://${encoded_login}:${token}@${repo}
    git clone --verbose --mirror $repo
    # Stop on error
    set -o errexit
done

# Now we can do an update on the ones we already have
for dir in `find . -maxdepth 1 -type d -name '*.git'`
do
    pushd "$dir" > /dev/null
    if [ "$verbose" != '' ];then echo "Updating " $dir;fi
    git remote update
    popd > /dev/null
done
popd > /dev/null

# ORGANIZATIONS

# Get the list of top_level groups the user is member of
group_ids=$(curl --silent --header "PRIVATE-TOKEN: ${token}" ${api_url}"/groups?top_level_only=true&per_page=5000"| jq '.[].id'|sort -n)

# Loop through groups, retrieve the repos, and store them in subdirectories
for group_id in $group_ids
do
    login=$(curl --silent --header "PRIVATE-TOKEN: ${token}" ${api_url}"/groups/${group_id}" | jq -r .name)
    if [ "$verbose" != '' ];then echo "Working on $login";fi
    
    list_of_repos=''
    list_of_repos=$(curl --silent --header "PRIVATE-TOKEN: ${token}" ${api_url}"/groups/${group_id}/projects?simple=true&per_page=50000"| jq -r '.[].'${method}'_to_repo')
    # Mirror the repo's
    mkdir -p "${backup_dir}/${login}"
    pushd "${backup_dir}/${login}" > /dev/null

    for repo in $list_of_repos
    do
        # We need to allow errors here, as git will give an error if the repo was already present
        set +o errexit
        if [ "$verbose" != '' ];then echo "Cloning " $repo;fi
        # Remove https://
        repo=$(echo ${repo} | sed 's|https://||')
        # encode login
        encoded_login=$(printf %s "${login}" | jq -s -R -r @uri)
        # Add auth
        repo=https://${encoded_login}:${token}@${repo}
        git clone --verbose --mirror "$repo"
        # Stop on error
        set -o errexit
    done

    for dir in `find . -maxdepth 1 -type d -name '*.git'`
    do
        pushd "$dir" > /dev/null
        if [ "$verbose" != '' ];then echo "Updating " $repo;fi
        git remote update
        popd > /dev/null
    done
    popd > /dev/null
done

gitmirror.sh

#!/bin/bash
# Github mirror script
# Jan van Haarst [email protected]
# 20150115

# Debug
#set -o xtrace
#set -o verbose

# Stop on error
set -o errexit

# Show some debug info
verbose=''

# Create a token at https://github.com/settings/tokens/new
token=''

if [ "$token" == '' ] 
then
    if [ "$GITHUB_TOKEN" != '' ]
    then
        token=$GITHUB_TOKEN
    else
        echo "Please enter a github token in the script, or run it like this:"
        echo "GITHUB_TOKEN=( token from https://github.com/settings/tokens/new ) " $(basename $0)
        echo "Optional : set the backup directory in the script, or pass BACKUP_DIR to it."
        echo "Otherwise the script will the working directory."
        exit 1
    fi
fi

# Place to store the mirrors
backup_dir=''

if [ "$backup_dir" == '' ] 
then
    if [ "$BACKUP_DIR" != '' ]
    then
        backup_dir="$BACKUP_DIR"
    else
        backup_dir="$PWD"
    fi
fi

# Github API URL
api_url="https://api.github.com"

# What method do we use to login and retrieve the repos ?
method="ssh_url"

# Check to see if we have jq available
if ! command -v jq > /dev/null 2>&1; then
    echo "This script needs jq (https://stedolan.github.io/jq/) to function."
    echo "Please install, and try again."
    exit 1
fi

# Get the right URL for the "current_user_url"
current_user_url=$(curl -s -u ${token}:x-oauth-basic https://api.github.com | jq --raw-output '.current_user_url')

# Get the right URL for the "organization_repositories_url"
organization_repositories_url=$(curl -s -u ${token}:x-oauth-basic https://api.github.com | jq --raw-output '.organization_repositories_url')

# What is the login that belongs to this token ?
login=$(curl -s -u ${token}:x-oauth-basic $current_user_url | jq --raw-output '.login')

# Get the repos_url and organizations_url of this user
repos_url=$(curl -s -u ${token}:x-oauth-basic $current_user_url | jq --raw-output '.repos_url')

# Get the user repo's
# To be sure we get all repo's, we loop a couple of times
list_of_repos=''
for counter in `seq 1 100`
do
    # We need to allow errors here, as the jq will return an errorcode if no match is found
    set +o errexit
    repos=$(curl -s -u ${token}:x-oauth-basic ${repos_url}"?page=${counter}&per_page=100" | jq --raw-output '.[].'${method}'')
    # Stop on error
    set -o errexit
    if [ "$repos" == '' ]
    then
        break
    fi
    list_of_repos="$list_of_repos $repos"
done

# Mirror the repo's
mkdir -p "${backup_dir}/${login}"
pushd "${backup_dir}/${login}" > /dev/null

for repo in $list_of_repos
do
    # We need to allow errors here, as git will give an error if the repo was already present
    set +o errexit
    if [ "$verbose" != '' ];then echo "Cloning " $repo;fi
    git clone --mirror "$repo"
    # Stop on error
    set -o errexit
done

# Now we can do an update on the ones we already have
for dir in `find . -maxdepth 1 -type d -name '*.git'`
do
    pushd "$dir" > /dev/null
    if [ "$verbose" != '' ];then echo "Updating " $dir;fi
    git remote update
    popd > /dev/null
done
popd > /dev/null

# ORGANIZATIONS

# Get the list of repos_url for the organizations the user is member of
ORGS_logins=$(curl -s -u ${token}:x-oauth-basic https://api.github.com/user/orgs | jq --raw-output '.[].login' )

# Loop through organizations, retrieve the repos, and store them in subdirectories
for login in $ORGS_logins
do
    if [ "$verbose" != '' ];then echo echo "Working on $login";fi
    organization_repos_url=$(curl -s -u ${token}:x-oauth-basic https://api.github.com/orgs/${login} |  jq --raw-output '.repos_url' )
    list_of_repos=''
    for counter in `seq 1 100`
    do
        set +o errexit
        repos=$(curl -s -u ${token}:x-oauth-basic ${organization_repos_url}"?page=${counter}&per_page=100" | jq --raw-output '.[].'${method}'')
        set -o errexit
        if [ "$repos" == '' ]
        then
            break
        fi
        list_of_repos="$list_of_repos $repos"
    done
    mkdir -p "${backup_dir}/${login}"
    pushd "${backup_dir}/${login}" > /dev/null
    for repo in $list_of_repos
    do
        if [ "$verbose" != '' ];then echo "Cloning " $repo;fi
        set +o errexit
        git clone --mirror "$repo"
        set -o errexit
    done
    for dir in `find . -maxdepth 1 -type d -name '*.git'`
    do
        pushd "$dir" > /dev/null
        if [ "$verbose" != '' ];then echo "Updating " $repo;fi
        git remote update
        popd > /dev/null
    done
    popd > /dev/null
done