Clones and or pulls the latest updates Org Repos into current working directory. If you just want to go down all the directories you already have and pull the latest from the current branches use this instead: find . -type d -depth 1 -exec git --git-dir={}/.git --work-tree=$PWD/{} pull origin master \;

A_script_to_Git_all_the_repos.md

Get all of the repos

It downloads all of the repos for a person or organization.

Dockerfile

FROM python:3.8-alpine
# DOCKER_BUILDKIT=1 docker build --progress=plain --no-cache -t gitallreposimage -f $(pwd)/Dockerfile --build-arg USER_ID=$(id -u) --build-arg GROUP_ID=$(id -g) .
# docker run --rm -it -v $(pwd):/app --name git_all_repos gitallreposimage --secret id=env_keys,src=~/.ssh/env_keys chmod ugo+rwx /app --user $(id -u):$(id -g)

RUN apk update && apk upgrade && \
    apk add --no-cache bash git openssh su-exec sudo
RUN ln -s /usr/bin/python3 /usr/bin/python

RUN mkdir -p /app
WORKDIR /app
RUN chmod -R 777 /app

ARG USER_ID
ARG GROUP_ID
ENV USR=appuser
ENV GRP=appgroup

RUN set -x ; addgroup -g "$GROUP_ID" -S "$GRP" && \
	adduser \
	--disabled-password \
	-g "$GROUP_ID" \
	-D \
	-s "/bin/bash" \
	-h "/home/$USR" \
	-u "$USER_ID" \
	-G "$GRP" "$USR" && exit 0 ; exit 1

RUN echo "$USR ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/$USR && chmod 0440 /etc/sudoers.d/$USR

USER $USR

# RUN pip install --upgrade setuptools pip wheel pipenv

# Run the executable
ENTRYPOINT ["/app/Git_all_repos.sh"]

git_all_repos.py

#!/usr/bin/env python3
import sys,os, time
# from tkinter import PAGES

try:
  import requests
except ImportError:
  print("Trying to Install required module: requests\n")
  os.system('python -m pip install requests')
import requests

org = sys.argv[1]

def fetch_repos(PAGE):
    giturl = "https://api.github.com/orgs/%s/repos?page=%d?per_page=100"
    if len(sys.argv) > 1:
        _dir = os.getcwd()
        if not os.path.exists(os.path.join(os.path.sep,_dir,'Orgs/', '%s' % org)):
            os.system("mkdir -p Orgs/"+org)
        r = requests.get(giturl %(org, PAGE))
        if r.status_code != 200:
            # x-ratelimit-used
            limit = r.headers['x-RaTeLiMiT-lImIt']
            # x-ratelimit-reset: 1645462141
            reset = int(r.headers['x-RaTeLiMiT-rEsEt'])
            while int(time.time()) < reset:
                print("Rate Limit Exceeded! Sleeping for 5 minutes; Reset Time: %s" % (time.ctime(reset)))
                time.sleep(5 * 60)
            r = requests.get(giturl %(org, PAGE))
        if r.status_code == 200:
            rdata = r.json()
            for repo in rdata:
                print(("Got %d repositories for the Org: %s\nDownloading %s" % (len(rdata),org,repo['name'])) )
                if not os.path.exists(os.path.join(os.path.sep,_dir,'Orgs/%s/%s' % (org, repo['name']))):
                    print("Cloning %s" % repo['name'])
                    os.system("cd Orgs/" + org + " && git clone " + repo['clone_url'])
                else:
                    print("Pulling %s" % repo['name'])
                    os.system("cd Orgs/" + org +"/"+ repo['name'] + " && git reset --hard && git fetch --all && echo "+repo['name']+"&& echo ''" )
    else:
        print(("Usage: %s git_hub_org_name" % (sys.argv[0])))

    if len(rdata) > PAGE*100:
        fetch_repos(PAGE+1)

fetch_repos(1)

Git_all_repos.sh

#!/usr/bin/env bash

# Array of Github Organizations/People to clone/pull
# @type {[type]}
#
# This is intended to be a scheduled automated process to backup repos

sudo chown -R $(id -u):$(id -g) /home/$(whoami) /usr/local/
sudo chmod -R 775 /home/$(whoami) /usr/local/lib/ /usr/local/bin

# Array of Organizations
# org_array=(lyrasis)
org_array=$(<./org_list.txt)
python_binary=$(which python3)
# Reset pipenv every time.

pip install --upgrade setuptools pip wheel
pip install pipenv
pip install --upgrade pipenv

pip list | grep pipenv
if [ $? -ne 0 ]; then
    echo "pipenv failed to install; exiting $?"
    exit 1
fi

python3 -m pipenv --python $python_binary --rm
python3 -m pipenv --python $python_binary sync

python3 -m pipenv --python $python_binary run python git_all_users_from_islandora.py

if [ ! -d "Orgs" ]; then
  mkdir -p Orgs
fi

for i in "${org_array[@]}"
do
  echo -e "\n\n\n\nProcessing $i"
  python3 -m pipenv --python $python_binary run python git_all_repos.py $i
done

# Array of People
# people_array=(DonRichards CanOfBees cdeaneGit dgreene-utk kzayasru markpbaggett pc37utn robert-patrick-waltz winter23 bryjbrown dannylamb DiegoPino dmoses dwilcox jonathangreen jordandukart manez mjordan qadan rosiel ruebot willtp87 whikloj mrmiguez jessfraz MikeBarnhardt Alucidisland davidbasswwu)
people_array=$(<./ppl_list.txt)

if [ ! -d "Users" ]; then
  mkdir -p Users
fi

for j in "${people_array[@]}"
do
  python3 -m pipenv --python $python_binary run python git_all_User_repos.py $j
done

python3 -m pipenv --python $python_binary run python git_all_users_from_islandora.py --token $GITHUB_TOKEN
# python3 -m pipenv --python $python_binary run python git_all_users_from_islandora.py --token $GITHUB_TOKEN --org islandora-Labs

date >> last_time_git_all_repos_ran.txt

git_all_User_repos.py

#!/usr/bin/env python3
# Download all the repositories from a github user
import sys,os

try:
  import requests
except ImportError:
  print("Trying to Install required module: requests\n")
  os.system('python -m pip install requests')
import requests

giturl = "https://api.github.com/users/%s/repos?per_page=100"
if len(sys.argv) > 1:
    _dir = os.getcwd()
    if not os.path.exists(os.path.join(os.path.sep,_dir,'Users','%s' % sys.argv[1])):
        os.system("mkdir Users/"+sys.argv[1])
    user = sys.argv[1]
    r = requests.get(giturl %(user))
    if r.status_code != 200:
        # x-ratelimit-used
        limit = r.headers['x-RaTeLiMiT-lImIt']
        # x-ratelimit-reset: 1645462141
        reset = int(r.headers['x-RaTeLiMiT-rEsEt'])
        while int(time.time()) < reset:
            print("Rate Limit Exceeded! Sleeping for 5 minutes; Reset Time: %s" % (time.ctime(reset)))
            time.sleep(5 * 60)
        r = requests.get(giturl %(user))
    if r.status_code == 200:
        rdata = r.json()
        for repo in rdata:
            print(("Got %d repositories for the User: %s\nDownloading %s" % (len(rdata),user,repo['name'])) )
            if not os.path.exists(os.path.join(os.path.sep,_dir,'Users',user,'%s' % repo['name'])):
                os.system("cd Users/" + user + " && git clone " + repo['clone_url'])
            else:
                os.system("cd Users/" + user +"/"+ repo['name'] + " && git reset --hard && git fetch --all && echo "+repo['name']+"&& echo ''" )
else:
    print(("Usage: %s git_hub_user_name" % (sys.argv[0]) ))

git_all_users_from_islandora.py

#!/usr/bin/env python3

from __future__ import print_function
import re
import json
import argparse
from collections import defaultdict
import os

try:
  import requests
except ImportError:
  print("Trying to Install required module: requests\n")
  os.system('python -m pip install requests')
import requests

parser = argparse.ArgumentParser(
    description="find non-members of a GitHub org by contributions to repos")
parser.add_argument("--token", nargs="?", type=str, help="GitHub API token")
parser.add_argument("--org", nargs="?", type=str, default="islandora",
                    help="the GitHub org to find member contirbutions for")

def get_pagination(response):
  if "Link" not in response.headers:
    return None
  parts = response.headers["Link"].split(",")
  res = {}
  for part in parts:
    match = re.search('<(.*)>; rel="(.*)"', part)
    if match:
      res[match.group(2)] = match.group(1)
  return res

def get_all_paginated(url, token=None):
    res = []
    while True:
        headers = {"Accept": "application/vnd.github.v3+json"}
        if token is not None:
            headers["Authorization"] = "token "+token
        r = requests.get(url, headers=headers, params={"per_page": "100"})
        res.extend(r.json())
        pagination = get_pagination(r)
        if pagination is None or "next" not in pagination:
            break
        url = pagination["next"]
    return res

def get_members(org, token=None):
    url = "https://api.github.com/orgs/%s/members" % org
    return get_all_paginated(url, token)


def get_contributors(repo, token=None):
    url = "https://api.github.com/repos/%s/stats/contributors" % repo
    return get_all_paginated(url, token)

def get_repos(org, token=None):
    url = "https://api.github.com/orgs/%s/repos" % org
    return get_all_paginated(url, token)

def main(args):
    org = "islandora"
    members_json = get_members(org, args.token)
    # print(json.dumps(members_json, sort_keys=True, indent=4))
    members = set(m["login"] for m in members_json)
    for member in members:
        os.system("pipenv run python git_all_User_repos.py %s" % member)

if __name__ == "__main__":
    args = parser.parse_args()
    main(args)

org_list.txt

islandora
Islandora-Labs
islandora-interest-groups

ppl_list.txt

DonRichards