Clones and or pulls the latest updates Org Repos into current working directory. If you just want to go down all the directories you already have and pull the latest from the current branches use this instead: find . -type d -depth 1 -exec git --git-dir={}/.git --work-tree=$PWD/{} pull origin master \;

A_script_to_Git_all_the_repos.md

Get all of the repos

It downloads all of the repos for a person or organization.

Dockerfile

FROM python:3.8-alpine
# DOCKER_BUILDKIT=1 docker build --progress=plain --no-cache -t gitallreposimage -f $(pwd)/Dockerfile --build-arg USER_ID=$(id -u) --build-arg GROUP_ID=$(id -g) .
# docker run --rm -it -v $(pwd):/app --name git_all_repos gitallreposimage --secret id=env_keys,src=~/.ssh/env_keys chmod ugo+rwx /app --user $(id -u):$(id -g)

RUN apk update && apk upgrade && \
    apk add --no-cache bash git openssh su-exec sudo
RUN ln -s /usr/bin/python3 /usr/bin/python

RUN mkdir -p /app
WORKDIR /app
RUN chmod -R 777 /app

ARG USER_ID
ARG GROUP_ID
ENV USR=appuser
ENV GRP=appgroup

RUN set -x ; addgroup -g "$GROUP_ID" -S "$GRP" && \
	adduser \
	--disabled-password \
	-g "$GROUP_ID" \
	-D \
	-s "/bin/bash" \
	-h "/home/$USR" \
	-u "$USER_ID" \
	-G "$GRP" "$USR" && exit 0 ; exit 1

RUN echo "$USR ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/$USR && chmod 0440 /etc/sudoers.d/$USR

USER $USR

# RUN pip install --upgrade setuptools pip wheel pipenv

# Run the executable
ENTRYPOINT ["/app/Git_all_repos.sh"]

git_all_repos.py

#!/usr/bin/env python3
import sys,os, time
# from tkinter import PAGES

try:
  import requests
except ImportError:
  print("Trying to Install required module: requests\n")
  os.system('python -m pip install requests')
import requests

org = sys.argv[1]

def fetch_repos(PAGE):
    giturl = "https://api.github.com/orgs/%s/repos?page=%d?per_page=100"
    if len(sys.argv) > 1:
        _dir = os.getcwd()
        if not os.path.exists(os.path.join(os.path.sep,_dir,'Orgs/', '%s' % org)):
            os.system("mkdir -p Orgs/"+org)
        r = requests.get(giturl %(org, PAGE))
        if r.status_code != 200:
            # x-ratelimit-used
            limit = r.headers['x-RaTeLiMiT-lImIt']
            # x-ratelimit-reset: 1645462141
            reset = int(r.headers['x-RaTeLiMiT-rEsEt'])
            while int(time.time()) < reset:
                print("Rate Limit Exceeded! Sleeping for 5 minutes; Reset Time: %s" % (time.ctime(reset)))
                time.sleep(5 * 60)
            r = requests.get(giturl %(org, PAGE))
        if r.status_code == 200:
            rdata = r.json()
            for repo in rdata:
                print(("Got %d repositories for the Org: %s\nDownloading %s" % (len(rdata),org,repo['name'])) )
                if not os.path.exists(os.path.join(os.path.sep,_dir,'Orgs/%s/%s' % (org, repo['name']))):
                    print("Cloning %s" % repo['name'])
                    os.system("cd Orgs/" + org + " && git clone " + repo['clone_url'])
                else:
                    print("Pulling %s" % repo['name'])
                    os.system("cd Orgs/" + org +"/"+ repo['name'] + " && git reset --hard && git fetch --all && echo "+repo['name']+"&& echo ''" )
    else:
        print(("Usage: %s git_hub_org_name" % (sys.argv[0])))

    if len(rdata) > PAGE*100:
        fetch_repos(PAGE+1)

fetch_repos(1)

Git_all_repos.sh

#!/usr/bin/env bash

# Array of Github Organizations/People to clone/pull
# @type {[type]}
#
# This is intended to be a scheduled automated process to backup repos


# Use this to run the script from the command line on a MAC.
# - python3 -m pipenv --python $python_binary run python git_all_users_from_islandora.py

# Otherwise use this to run the script from the command line on a linux machine.
# - pipenv run python git_all_users_from_islandora.py


echo ""
echo ""
echo " -------------------------------------------------------------------------- "
echo " -              Running Github rate limit check                           - "
echo " -------------------------------------------------------------------------- "
echo ""

if ! [ -x "$(command -v jq)" ]; then
    echo 'Error: jq is not installed.' >&2
    exit 1
    # apk add jq --quiet
    # apt-get install jq
fi

: '
Estimate the number of requests that will be made during the use of this script.
Too little requests will cause the script to fail.
Too many requests will cause the script to take a long time to complete.
'
ESTIMATED_QUERIES_NEEDED=450

function check_github_rate_limit() {
    REMAINING=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GITHUB_TOKEN" https://api.github.com/rate_limit | jq .rate.remaining)

    # While remaining request count is too low to complete this cycle, sleep for 30 seconds and check again.
    while [ "$REMAINING" -lt "$ESTIMATED_QUERIES_NEEDED" ]; do
        REMAINING=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GITHUB_TOKEN" https://api.github.com/rate_limit | jq .rate.remaining)
        GH=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GITHUB_TOKEN" https://api.github.com/rate_limit | jq .rate.reset)
        now=$(date '+%s')
        echo ""
        echo ""
        echo " -------------------------------------------------------------------------- "
        echo " - Github remaining request count: $REMAINING "
        echo " - Github's hourly rate limit is close to going over. "
        echo " - Waiting $(( $(( $GH - $now )) / 60 )) minutes for Github rate limit to reset."
        echo " -------------------------------------------------------------------------- "
        sleep 30
    done
    echo " -------------------------------------------------------------------------- "
    echo ""
}

# sudo chown -R $(id -u):$(id -g) /home/$(whoami) /usr/local/
# sudo chmod -R 775 /home/$(whoami) /usr/local/lib/ /usr/local/bin

# Array of Organizations
# org_array=(lyrasis)
org_array=$(<./org_list.txt)
python_binary=$(which python3)
# Reset pipenv every time.

pip install --user --upgrade setuptools pip wheel
pip install --user pipenv
pip install --user --upgrade pipenv
pipenv install --dev --python 3.8

pip list | grep pipenv
if [ $? -ne 0 ]; then
    echo "pipenv failed to install; exiting $?"
    exit 1
fi

# python3 -m pipenv --python $python_binary --rm
# python3 -m pipenv --python $python_binary sync
pipenv --rm
pipenv sync

# python3 -m pipenv --python $python_binary run python git_all_users_from_islandora.py
pipenv run python git_all_users_from_islandora.py

if [ ! -d "Orgs" ]; then
  mkdir -p Orgs
fi

for i in "${org_array[@]}"
do
  echo -e "\n\n\n\nProcessing $i"
  check_github_rate_limit
  # python3 -m pipenv --python $python_binary run python git_all_repos.py $i
  pipenv run python git_all_repos.py $i
done

# Array of People
# people_array=(DonRichards CanOfBees cdeaneGit dgreene-utk kzayasru markpbaggett pc37utn robert-patrick-waltz winter23 bryjbrown dannylamb DiegoPino dmoses dwilcox jonathangreen jordandukart manez mjordan qadan rosiel ruebot willtp87 whikloj mrmiguez jessfraz MikeBarnhardt Alucidisland davidbasswwu)
people_array=$(<./ppl_list.txt)

if [ ! -d "Users" ]; then
  mkdir -p Users
fi

for j in "${people_array[@]}"
do
  check_github_rate_limit
  # python3 -m pipenv --python $python_binary run python git_all_User_repos.py $j
  pipenv run python git_all_User_repos.py $j
done

check_github_rate_limit
# python3 -m pipenv --python $python_binary run python git_all_users_from_islandora.py --token $GITHUB_TOKEN
pipenv run python git_all_users_from_islandora.py --token $GITHUB_TOKEN
# python3 -m pipenv --python $python_binary run python git_all_users_from_islandora.py --token $GITHUB_TOKEN --org islandora-Labs

date >> last_time_git_all_repos_ran.txt

git_all_User_repos.py

#!/usr/bin/env python3
# Download all the repositories from a github user
import sys,os

try:
  import requests
except ImportError:
  print("Trying to Install required module: requests\n")
  os.system('python -m pip install requests')
import requests

giturl = "https://api.github.com/users/%s/repos?per_page=100"
if len(sys.argv) > 1:
    _dir = os.getcwd()
    if not os.path.exists(os.path.join(os.path.sep,_dir,'Users','%s' % sys.argv[1])):
        os.system("mkdir Users/"+sys.argv[1])
    user = sys.argv[1]
    r = requests.get(giturl %(user))
    if r.status_code != 200:
        # x-ratelimit-used
        limit = r.headers['x-RaTeLiMiT-lImIt']
        # x-ratelimit-reset: 1645462141
        reset = int(r.headers['x-RaTeLiMiT-rEsEt'])
        while int(time.time()) < reset:
            print("Rate Limit Exceeded! Sleeping for 5 minutes; Reset Time: %s" % (time.ctime(reset)))
            time.sleep(5 * 60)
        r = requests.get(giturl %(user))
    if r.status_code == 200:
        rdata = r.json()
        for repo in rdata:
            print(("Got %d repositories for the User: %s\nDownloading %s" % (len(rdata),user,repo['name'])) )
            if not os.path.exists(os.path.join(os.path.sep,_dir,'Users',user,'%s' % repo['name'])):
                os.system("cd Users/" + user + " && git clone " + repo['clone_url'])
            else:
                os.system("cd Users/" + user +"/"+ repo['name'] + " && git reset --hard && git fetch --all && echo "+repo['name']+"&& echo ''" )
else:
    print(("Usage: %s git_hub_user_name" % (sys.argv[0]) ))

git_all_users_from_islandora.py

#!/usr/bin/env python3

from __future__ import print_function
import re
import json
import argparse
from collections import defaultdict
import os

try:
  import requests
except ImportError:
  print("Trying to Install required module: requests\n")
  os.system('python -m pip install requests')
import requests

parser = argparse.ArgumentParser(
    description="find non-members of a GitHub org by contributions to repos")
parser.add_argument("--token", nargs="?", type=str, help="GitHub API token")
parser.add_argument("--org", nargs="?", type=str, default="islandora",
                    help="the GitHub org to find member contirbutions for")

def get_pagination(response):
  if "Link" not in response.headers:
    return None
  parts = response.headers["Link"].split(",")
  res = {}
  for part in parts:
    match = re.search('<(.*)>; rel="(.*)"', part)
    if match:
      res[match.group(2)] = match.group(1)
  return res

def get_all_paginated(url, token=None):
    res = []
    while True:
        headers = {"Accept": "application/vnd.github.v3+json"}
        if token is not None:
            headers["Authorization"] = "token "+token
        r = requests.get(url, headers=headers, params={"per_page": "100"})
        res.extend(r.json())
        pagination = get_pagination(r)
        if pagination is None or "next" not in pagination:
            break
        url = pagination["next"]
    return res

def get_members(org, token=None):
    url = "https://api.github.com/orgs/%s/members" % org
    return get_all_paginated(url, token)


def get_contributors(repo, token=None):
    url = "https://api.github.com/repos/%s/stats/contributors" % repo
    return get_all_paginated(url, token)

def get_repos(org, token=None):
    url = "https://api.github.com/orgs/%s/repos" % org
    return get_all_paginated(url, token)

def main(args):
    org = "islandora"
    members_json = get_members(org, args.token)
    # print(json.dumps(members_json, sort_keys=True, indent=4))
    members = set(m["login"] for m in members_json)
    for member in members:
        os.system("pipenv run python git_all_User_repos.py %s" % member)

if __name__ == "__main__":
    args = parser.parse_args()
    main(args)

org_list.txt

islandora
Islandora-Labs
islandora-interest-groups

ppl_list.txt

DonRichards