Skip to content

Instantly share code, notes, and snippets.

@philfreo
Last active September 11, 2026 16:16
Show Gist options
  • Select an option

  • Save philfreo/7e5101edff3e2f0a5398b3ad5fe13232 to your computer and use it in GitHub Desktop.

Select an option

Save philfreo/7e5101edff3e2f0a5398b3ad5fe13232 to your computer and use it in GitHub Desktop.
Clone All GitHub Repos for an Organization
ORG=${1:?Usage: $0 <org>}
gh repo list $ORG --no-archived --limit 4000 | while read -r repo _; do
gh repo clone "$repo"
done
#!/usr/bin/env bash
#
# pull-all-repos.sh — clone new org repos, bring existing clones up to date,
# retire archived ones, and clean up dead empty ones.
#
# Read-only by default. Prints a report and changes nothing until you pass --apply.
#
# Never: reset, checkout, branch switch, clean, stash, rebase, push, or force.
# Existing clones are only ever fast-forwarded, and only when that cannot lose work.
#
# archived upstream -> fast-forwarded one last time, then moved to _archived/
# deleted upstream, no commits, no files -> directory removed (nothing to lose)
# deleted upstream, has any content -> warned about only, never touched
#
# A repo is only ever treated as deleted when the GitHub API returns 404 for it.
# A failed fetch alone is never enough: that could just be the network.
set -uo pipefail
ORG=closeio
JOBS=12
APPLY=0
FETCH=1
AUDIT=1
PRUNE_DEAD=1
ARCHIVE_DIR=_archived
EXCLUDE_DIRS="_archived .claude"
usage() {
cat <<'USAGE'
usage: ./pull-all-repos.sh [options]
--apply clone, fast-forward, retire archived, remove dead empties
--org NAME GitHub org (default: closeio)
--jobs N parallel fetch/clone jobs (default: 12)
--no-fetch skip 'git fetch'; use existing remote-tracking refs (offline)
--no-audit skip the API check for archived/renamed/deleted repos
(also disables retiring and pruning)
--no-prune-deleted report empty clones of deleted repos, but do not remove them
-h, --help this
USAGE
}
while [ $# -gt 0 ]; do
case "$1" in
--apply) APPLY=1 ;;
--org) ORG=${2:?--org needs a value}; shift ;;
--jobs) JOBS=${2:?--jobs needs a value}; shift ;;
--no-fetch) FETCH=0 ;;
--no-audit) AUDIT=0 ;;
--no-prune-deleted) PRUNE_DEAD=0 ;;
-h|--help) usage; exit 0 ;;
*) echo "unknown option: $1" >&2; usage >&2; exit 2 ;;
esac
shift
done
die() { echo "error: $*" >&2; exit 1; }
command -v git >/dev/null || die "git not found"
command -v gh >/dev/null || die "gh not found"
gh auth status >/dev/null 2>&1 || die "gh is not authenticated; run: gh auth login"
TMP=$(mktemp -d) || die "mktemp failed"
trap 'rm -rf "$TMP"' EXIT
for f in remote alldirs repos notgit missing strays fetchfail archived gone renamed foreign report; do : > "$TMP/$f"; done
if [ "$APPLY" -eq 1 ]; then MODE="APPLY — will clone, fast-forward, retire, and prune"
else MODE="DRY RUN — nothing will be changed (pass --apply to act)"; fi
echo "=> $MODE"
echo "=> org: $ORG dir: $(pwd)"
[ "$APPLY" -eq 0 ] && [ "$FETCH" -eq 1 ] && \
echo "=> note: dry run still runs 'git fetch' so the report is accurate."
echo "=> fetch only updates remote-tracking refs; it never touches your"
echo "=> working tree, local branches, commits, or stashes."
echo
# ---------------------------------------------------------------- remote list
echo "=> listing non-archived repos in $ORG ..."
gh repo list "$ORG" --no-archived --limit 4000 --json name --jq '.[].name' \
2>/dev/null | sort -u > "$TMP/remote" || die "gh repo list failed"
REMOTE_N=$(grep -c . < "$TMP/remote")
[ "$REMOTE_N" -ge 10 ] || die "only $REMOTE_N repos returned — refusing to act on a suspicious list"
echo " $REMOTE_N repos"
# ---------------------------------------------------------------- local dirs
{ ls -d -- */ 2>/dev/null; ls -d -- .*/ 2>/dev/null; } \
| sed 's#/$##' | grep -vx -e '.' -e '..' | sort -u > "$TMP/alldirs"
for x in $EXCLUDE_DIRS; do grep -vx -- "$x" "$TMP/alldirs" > "$TMP/t" && mv "$TMP/t" "$TMP/alldirs"; done
while IFS= read -r d; do
if [ -e "$d/.git" ]; then echo "$d" >> "$TMP/repos"; else echo "$d" >> "$TMP/notgit"; fi
done < "$TMP/alldirs"
echo " $(grep -c . < "$TMP/repos") local clones, $(grep -c . < "$TMP/notgit") non-repo dirs"
echo
# ---------------------------------------------------------------- phase 1: clone
comm -23 "$TMP/remote" "$TMP/alldirs" > "$TMP/missing"
MISSING_N=$(grep -c . < "$TMP/missing")
echo "== new repos in $ORG not cloned here: $MISSING_N"
if [ "$MISSING_N" -gt 0 ]; then
sed 's/^/ + /' "$TMP/missing"
if [ "$APPLY" -eq 1 ]; then
echo " cloning ..."
xargs -P "$JOBS" -I{} sh -c \
'gh repo clone "'"$ORG"'/$1" -- --quiet </dev/null 2>&1 | sed "s|^| clone $1: |"' _ {} \
< "$TMP/missing"
{ cat "$TMP/repos"; while IFS= read -r n; do [ -e "$n/.git" ] && echo "$n"; done < "$TMP/missing"; } \
| sort -u > "$TMP/t" && mv "$TMP/t" "$TMP/repos"
fi
fi
echo
# ---------------------------------------------------------------- phase 2: fetch
if [ "$FETCH" -eq 1 ]; then
echo "== fetching $(grep -c . < "$TMP/repos") clones (${JOBS} at a time) ..."
PRUNE_OPT=""; [ "$APPLY" -eq 1 ] && PRUNE_OPT="--prune"
# a failed fetch is recorded, not ignored: we must not report a repo whose
# remote is unreachable as "up to date" just because its stale refs match
xargs -P "$JOBS" -I{} sh -c \
'err=$(git -C "$1" fetch --all '"$PRUNE_OPT"' --quiet 2>&1) || {
printf "%s\t%s\n" "$1" "$(printf "%s" "$err" | grep -iE "not found|denied|could not read|timed out|resolve host" | head -1)" >> "'"$TMP"'/fetchfail"
echo " fetch failed: $1"
}' _ {} < "$TMP/repos"
echo
fi
fetch_failed() { cut -f1 "$TMP/fetchfail" | grep -qx -- "$1"; }
fetch_error() { awk -F'\t' -v d="$1" '$1==d {print $2; exit}' "$TMP/fetchfail"; }
# ------------------------------------------------ phase 3: audit (archived/gone)
# runs before updating, so archived repos can be fast-forwarded and then retired
if [ "$AUDIT" -eq 1 ]; then
echo "== checking repos missing from the org list ..."
comm -13 "$TMP/remote" "$TMP/repos" > "$TMP/strays"
while IFS= read -r d; do
url=$(git -C "$d" remote get-url origin 2>/dev/null)
case "$url" in
*"github.com"[:/]"$ORG"/*) ;;
"") continue ;;
*) printf '%s\t%s\n' "$d" "$url" >> "$TMP/foreign"; continue ;;
esac
name=${url##*/}; name=${name%.git}
if ! info=$(gh api "repos/$ORG/$name" --jq '[.name,(.archived|tostring)]|join(" ")' 2>/dev/null) \
|| [ -z "$info" ]; then
echo "$d" >> "$TMP/gone"; continue
fi
real=${info%% *}; arch=${info##* }
if [ "$arch" = "true" ]; then echo "$d" >> "$TMP/archived"
elif [ "$real" != "$d" ]; then printf '%s\t%s\n' "$d" "$real" >> "$TMP/renamed"; fi
done < "$TMP/strays"
printf ' archived: %s deleted: %s renamed: %s\n\n' \
"$(grep -c . < "$TMP/archived")" "$(grep -c . < "$TMP/gone")" "$(grep -c . < "$TMP/renamed")"
fi
is_archived() { grep -qx -- "$1" "$TMP/archived"; }
is_gone() { grep -qx -- "$1" "$TMP/gone"; }
# ---------------------------------------------------------------- phase 4: update
note() { printf '%s\t%s\t%s\n' "$1" "$2" "$3" >> "$TMP/report"; }
# true only if the clone holds nothing a human could miss
holds_nothing() {
d=$1
git -C "$d" rev-parse -q --verify HEAD >/dev/null 2>&1 && return 1 # has commits
[ "$(ls -A "$d" | grep -vx '.git' | grep -c .)" -eq 0 ] || return 1 # has files
[ "$(git -C "$d" stash list 2>/dev/null | grep -c .)" -eq 0 ] || return 1
[ "$(git -C "$d" worktree list 2>/dev/null | grep -c .)" -le 1 ] || return 1
[ "$(git -C "$d" for-each-ref refs/heads refs/tags 2>/dev/null | grep -c .)" -eq 0 ] || return 1
return 0
}
# a repo deleted from GitHub: remove it only if it is a hollow shell
handle_deleted() {
d=$1
# paranoia: this is the only irreversible operation in the script
case "$d" in ""|.|..|/*|*/*) note DELETED "$d" "refusing to prune a suspicious path"; return ;; esac
[ -d "./$d/.git" ] || { note DELETED "$d" "refusing to prune: not a git clone"; return; }
if ! holds_nothing "$d"; then
note DELETED "$d" "gone from GitHub but has local content — untouched, may be the only copy"; return
fi
if [ "$PRUNE_DEAD" -eq 0 ]; then
note DELETED-EMPTY "$d" "gone from GitHub, empty clone — kept (--no-prune-deleted)"; return
fi
if [ "$APPLY" -eq 0 ]; then
note WOULD-REMOVE "$d" "gone from GitHub, no commits and no files — would delete the directory"; return
fi
if rm -rf "./$d"; then note REMOVED "$d" "gone from GitHub, empty clone — directory deleted"
else note DELETED "$d" "gone from GitHub, empty clone — delete failed"; fi
}
# echoes "TAG<TAB>detail"; fast-forwards in place when that is safe
classify() {
d=$1
url=$(git -C "$d" remote get-url origin 2>/dev/null)
[ -n "$url" ] || { printf 'NO-REMOTE\tno '\''origin'\'' remote — local-only repo'; return; }
if fetch_failed "$d"; then printf 'UNREACHABLE\tfetch failed (%s) — state is stale, not verified' "$(fetch_error "$d")"; return; fi
branch=$(git -C "$d" symbolic-ref --short -q HEAD)
[ -n "$branch" ] || { printf 'DETACHED\tdetached HEAD — resolve by hand'; return; }
# '@{u}' does not resolve on an unborn branch, so fall back to the config
up=$(git -C "$d" rev-parse --abbrev-ref --symbolic-full-name '@{u}' 2>/dev/null)
if [ -z "$up" ] || [ "$up" = '@{u}' ]; then
ur=$(git -C "$d" config --get "branch.$branch.remote" 2>/dev/null)
um=$(git -C "$d" config --get "branch.$branch.merge" 2>/dev/null)
if [ -n "$ur" ] && [ -n "$um" ]; then up="$ur/${um#refs/heads/}"; else up=""; fi
fi
[ -n "$up" ] || { printf 'NO-UPSTREAM\tbranch '\''%s'\'' tracks nothing' "$branch"; return; }
has_head=1; git -C "$d" rev-parse -q --verify HEAD >/dev/null 2>&1 || has_head=0
if ! git -C "$d" rev-parse -q --verify "$up^{commit}" >/dev/null 2>&1; then
if [ "$has_head" -eq 0 ]; then printf 'EMPTY-BOTH\tno commits locally or on %s — nothing to sync' "$up"
else printf 'UPSTREAM-GONE\t%s -> %s no longer exists on the remote' "$branch" "$up"; fi
return
fi
# tracked modifications block us; untracked files do not (git refuses to
# overwrite an untracked file during a merge, so it fails safe on its own)
if [ -n "$(git -C "$d" status --porcelain --untracked-files=no 2>/dev/null)" ]; then
printf 'DIRTY\tuncommitted changes to tracked files on '\''%s'\''' "$branch"; return
fi
if [ "$has_head" -eq 1 ]; then
counts=$(git -C "$d" rev-list --left-right --count "$up...HEAD" 2>/dev/null)
behind=${counts%%[!0-9]*}; ahead=${counts##*[!0-9]}
if [ "${ahead:-0}" -gt 0 ] && [ "${behind:-0}" -gt 0 ]; then
printf 'DIVERGED\t%s is %s ahead / %s behind %s — left alone' "$branch" "$ahead" "$behind" "$up"; return
elif [ "${ahead:-0}" -gt 0 ]; then
printf 'AHEAD\t%s has %s unpushed commit(s), nothing to pull' "$branch" "$ahead"; return
elif [ "${behind:-0}" -eq 0 ]; then
printf 'UP-TO-DATE\t%s' "$branch"; return
fi
action=FF; detail=$(printf '%s: %s commit(s) behind %s' "$branch" "$behind" "$up")
else
action=EMPTY-CLONE; detail=$(printf 'no commits — restores from %s' "$up")
fi
case "$branch" in main|master|develop|trunk) ;; *) detail="$detail [feature branch — default branch stays stale]" ;; esac
if [ "$APPLY" -eq 0 ]; then printf 'WOULD-%s\t%s' "$action" "$detail"
elif out=$(git -C "$d" merge --ff-only "$up" 2>&1); then printf '%s\t%s' "$action" "$detail"
else printf 'FF-BLOCKED\t%s' "$(printf '%s' "$out" | head -1)"; fi
}
# archived repos: capture the final state, then get them out of the way
retire() {
d=$1; how=$2
if [ "$(git -C "$d" worktree list 2>/dev/null | grep -c .)" -gt 1 ]; then
note ARCHIVE-SKIP "$d" "archived; $how — NOT moved: linked worktree(s) would break"; return
fi
if [ -e "$ARCHIVE_DIR/$d" ]; then
note ARCHIVE-SKIP "$d" "archived; $how — NOT moved: $ARCHIVE_DIR/$d already exists"; return
fi
if [ "$APPLY" -eq 0 ]; then
note WOULD-ARCHIVE "$d" "archived; $how — would move to $ARCHIVE_DIR/"; return
fi
mkdir -p "$ARCHIVE_DIR"
if mv "$d" "$ARCHIVE_DIR/$d"; then note ARCHIVED-MOVED "$d" "archived; $how — moved to $ARCHIVE_DIR/"
else note ARCHIVE-SKIP "$d" "archived; $how — mv failed"; fi
}
while IFS= read -r d; do
if is_gone "$d"; then handle_deleted "$d"; continue; fi
res=$(classify "$d")
tag=${res%% *}; det=${res#* }
if is_archived "$d"; then retire "$d" "$tag${det:+ ($det)}"; else note "$tag" "$d" "$det"; fi
done < "$TMP/repos"
while IFS=$'\t' read -r d real; do
[ -n "$d" ] && note RENAMED "$d" "repo was renamed to '$real' — you may end up with both"
done < "$TMP/renamed"
while IFS=$'\t' read -r d url; do
[ -n "$d" ] && note FOREIGN "$d" "belongs to another org: $url"
done < "$TMP/foreign"
# ---------------------------------------------------------------- report
echo "== results"
for tag in FF WOULD-FF EMPTY-CLONE WOULD-EMPTY-CLONE ARCHIVED-MOVED WOULD-ARCHIVE ARCHIVE-SKIP \
REMOVED WOULD-REMOVE DELETED-EMPTY DELETED FF-BLOCKED DIRTY DIVERGED AHEAD \
NO-UPSTREAM UPSTREAM-GONE EMPTY-BOTH DETACHED NO-REMOTE UNREACHABLE RENAMED FOREIGN; do
n=$(awk -F'\t' -v t="$tag" '$1==t' "$TMP/report" | grep -c .)
[ "$n" -eq 0 ] && continue
echo; echo "-- $tag ($n)"
awk -F'\t' -v t="$tag" '$1==t {printf " %-42s %s\n", $2, $3}' "$TMP/report"
done
echo; echo "-- summary"
printf ' %s up to date, %s updated/updatable, %s retired, %s pruned, %s skipped, %s new to clone\n' \
"$(awk -F'\t' '$1=="UP-TO-DATE"' "$TMP/report" | grep -c .)" \
"$(awk -F'\t' '$1~/^(WOULD-)?(FF|EMPTY-CLONE)$/' "$TMP/report" | grep -c .)" \
"$(awk -F'\t' '$1~/^(ARCHIVED-MOVED|WOULD-ARCHIVE)$/' "$TMP/report" | grep -c .)" \
"$(awk -F'\t' '$1~/^(REMOVED|WOULD-REMOVE)$/' "$TMP/report" | grep -c .)" \
"$(awk -F'\t' '$1~/^(DIRTY|DIVERGED|AHEAD|NO-UPSTREAM|UPSTREAM-GONE|DETACHED|NO-REMOTE|FF-BLOCKED|UNREACHABLE|ARCHIVE-SKIP)$/' "$TMP/report" | grep -c .)" \
"$MISSING_N"
if awk -F'\t' '$1=="DELETED"' "$TMP/report" | grep -q .; then
echo " WARNING: $(awk -F'\t' '$1=="DELETED"' "$TMP/report" | grep -c .) repo(s) with local content no longer exist on GitHub — your clone may be the only copy."
fi
[ "$APPLY" -eq 0 ] && echo " (dry run — re-run with --apply)"
exit 0
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment