Last active
September 11, 2026 16:16
-
-
Save philfreo/7e5101edff3e2f0a5398b3ad5fe13232 to your computer and use it in GitHub Desktop.
Clone All GitHub Repos for an Organization
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ORG=${1:?Usage: $0 <org>} | |
| gh repo list $ORG --no-archived --limit 4000 | while read -r repo _; do | |
| gh repo clone "$repo" | |
| done |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env bash | |
| # | |
| # pull-all-repos.sh — clone new org repos, bring existing clones up to date, | |
| # retire archived ones, and clean up dead empty ones. | |
| # | |
| # Read-only by default. Prints a report and changes nothing until you pass --apply. | |
| # | |
| # Never: reset, checkout, branch switch, clean, stash, rebase, push, or force. | |
| # Existing clones are only ever fast-forwarded, and only when that cannot lose work. | |
| # | |
| # archived upstream -> fast-forwarded one last time, then moved to _archived/ | |
| # deleted upstream, no commits, no files -> directory removed (nothing to lose) | |
| # deleted upstream, has any content -> warned about only, never touched | |
| # | |
| # A repo is only ever treated as deleted when the GitHub API returns 404 for it. | |
| # A failed fetch alone is never enough: that could just be the network. | |
| set -uo pipefail | |
| ORG=closeio | |
| JOBS=12 | |
| APPLY=0 | |
| FETCH=1 | |
| AUDIT=1 | |
| PRUNE_DEAD=1 | |
| ARCHIVE_DIR=_archived | |
| EXCLUDE_DIRS="_archived .claude" | |
| usage() { | |
| cat <<'USAGE' | |
| usage: ./pull-all-repos.sh [options] | |
| --apply clone, fast-forward, retire archived, remove dead empties | |
| --org NAME GitHub org (default: closeio) | |
| --jobs N parallel fetch/clone jobs (default: 12) | |
| --no-fetch skip 'git fetch'; use existing remote-tracking refs (offline) | |
| --no-audit skip the API check for archived/renamed/deleted repos | |
| (also disables retiring and pruning) | |
| --no-prune-deleted report empty clones of deleted repos, but do not remove them | |
| -h, --help this | |
| USAGE | |
| } | |
| while [ $# -gt 0 ]; do | |
| case "$1" in | |
| --apply) APPLY=1 ;; | |
| --org) ORG=${2:?--org needs a value}; shift ;; | |
| --jobs) JOBS=${2:?--jobs needs a value}; shift ;; | |
| --no-fetch) FETCH=0 ;; | |
| --no-audit) AUDIT=0 ;; | |
| --no-prune-deleted) PRUNE_DEAD=0 ;; | |
| -h|--help) usage; exit 0 ;; | |
| *) echo "unknown option: $1" >&2; usage >&2; exit 2 ;; | |
| esac | |
| shift | |
| done | |
| die() { echo "error: $*" >&2; exit 1; } | |
| command -v git >/dev/null || die "git not found" | |
| command -v gh >/dev/null || die "gh not found" | |
| gh auth status >/dev/null 2>&1 || die "gh is not authenticated; run: gh auth login" | |
| TMP=$(mktemp -d) || die "mktemp failed" | |
| trap 'rm -rf "$TMP"' EXIT | |
| for f in remote alldirs repos notgit missing strays fetchfail archived gone renamed foreign report; do : > "$TMP/$f"; done | |
| if [ "$APPLY" -eq 1 ]; then MODE="APPLY — will clone, fast-forward, retire, and prune" | |
| else MODE="DRY RUN — nothing will be changed (pass --apply to act)"; fi | |
| echo "=> $MODE" | |
| echo "=> org: $ORG dir: $(pwd)" | |
| [ "$APPLY" -eq 0 ] && [ "$FETCH" -eq 1 ] && \ | |
| echo "=> note: dry run still runs 'git fetch' so the report is accurate." | |
| echo "=> fetch only updates remote-tracking refs; it never touches your" | |
| echo "=> working tree, local branches, commits, or stashes." | |
| echo | |
| # ---------------------------------------------------------------- remote list | |
| echo "=> listing non-archived repos in $ORG ..." | |
| gh repo list "$ORG" --no-archived --limit 4000 --json name --jq '.[].name' \ | |
| 2>/dev/null | sort -u > "$TMP/remote" || die "gh repo list failed" | |
| REMOTE_N=$(grep -c . < "$TMP/remote") | |
| [ "$REMOTE_N" -ge 10 ] || die "only $REMOTE_N repos returned — refusing to act on a suspicious list" | |
| echo " $REMOTE_N repos" | |
| # ---------------------------------------------------------------- local dirs | |
| { ls -d -- */ 2>/dev/null; ls -d -- .*/ 2>/dev/null; } \ | |
| | sed 's#/$##' | grep -vx -e '.' -e '..' | sort -u > "$TMP/alldirs" | |
| for x in $EXCLUDE_DIRS; do grep -vx -- "$x" "$TMP/alldirs" > "$TMP/t" && mv "$TMP/t" "$TMP/alldirs"; done | |
| while IFS= read -r d; do | |
| if [ -e "$d/.git" ]; then echo "$d" >> "$TMP/repos"; else echo "$d" >> "$TMP/notgit"; fi | |
| done < "$TMP/alldirs" | |
| echo " $(grep -c . < "$TMP/repos") local clones, $(grep -c . < "$TMP/notgit") non-repo dirs" | |
| echo | |
| # ---------------------------------------------------------------- phase 1: clone | |
| comm -23 "$TMP/remote" "$TMP/alldirs" > "$TMP/missing" | |
| MISSING_N=$(grep -c . < "$TMP/missing") | |
| echo "== new repos in $ORG not cloned here: $MISSING_N" | |
| if [ "$MISSING_N" -gt 0 ]; then | |
| sed 's/^/ + /' "$TMP/missing" | |
| if [ "$APPLY" -eq 1 ]; then | |
| echo " cloning ..." | |
| xargs -P "$JOBS" -I{} sh -c \ | |
| 'gh repo clone "'"$ORG"'/$1" -- --quiet </dev/null 2>&1 | sed "s|^| clone $1: |"' _ {} \ | |
| < "$TMP/missing" | |
| { cat "$TMP/repos"; while IFS= read -r n; do [ -e "$n/.git" ] && echo "$n"; done < "$TMP/missing"; } \ | |
| | sort -u > "$TMP/t" && mv "$TMP/t" "$TMP/repos" | |
| fi | |
| fi | |
| echo | |
| # ---------------------------------------------------------------- phase 2: fetch | |
| if [ "$FETCH" -eq 1 ]; then | |
| echo "== fetching $(grep -c . < "$TMP/repos") clones (${JOBS} at a time) ..." | |
| PRUNE_OPT=""; [ "$APPLY" -eq 1 ] && PRUNE_OPT="--prune" | |
| # a failed fetch is recorded, not ignored: we must not report a repo whose | |
| # remote is unreachable as "up to date" just because its stale refs match | |
| xargs -P "$JOBS" -I{} sh -c \ | |
| 'err=$(git -C "$1" fetch --all '"$PRUNE_OPT"' --quiet 2>&1) || { | |
| printf "%s\t%s\n" "$1" "$(printf "%s" "$err" | grep -iE "not found|denied|could not read|timed out|resolve host" | head -1)" >> "'"$TMP"'/fetchfail" | |
| echo " fetch failed: $1" | |
| }' _ {} < "$TMP/repos" | |
| echo | |
| fi | |
| fetch_failed() { cut -f1 "$TMP/fetchfail" | grep -qx -- "$1"; } | |
| fetch_error() { awk -F'\t' -v d="$1" '$1==d {print $2; exit}' "$TMP/fetchfail"; } | |
| # ------------------------------------------------ phase 3: audit (archived/gone) | |
| # runs before updating, so archived repos can be fast-forwarded and then retired | |
| if [ "$AUDIT" -eq 1 ]; then | |
| echo "== checking repos missing from the org list ..." | |
| comm -13 "$TMP/remote" "$TMP/repos" > "$TMP/strays" | |
| while IFS= read -r d; do | |
| url=$(git -C "$d" remote get-url origin 2>/dev/null) | |
| case "$url" in | |
| *"github.com"[:/]"$ORG"/*) ;; | |
| "") continue ;; | |
| *) printf '%s\t%s\n' "$d" "$url" >> "$TMP/foreign"; continue ;; | |
| esac | |
| name=${url##*/}; name=${name%.git} | |
| if ! info=$(gh api "repos/$ORG/$name" --jq '[.name,(.archived|tostring)]|join(" ")' 2>/dev/null) \ | |
| || [ -z "$info" ]; then | |
| echo "$d" >> "$TMP/gone"; continue | |
| fi | |
| real=${info%% *}; arch=${info##* } | |
| if [ "$arch" = "true" ]; then echo "$d" >> "$TMP/archived" | |
| elif [ "$real" != "$d" ]; then printf '%s\t%s\n' "$d" "$real" >> "$TMP/renamed"; fi | |
| done < "$TMP/strays" | |
| printf ' archived: %s deleted: %s renamed: %s\n\n' \ | |
| "$(grep -c . < "$TMP/archived")" "$(grep -c . < "$TMP/gone")" "$(grep -c . < "$TMP/renamed")" | |
| fi | |
| is_archived() { grep -qx -- "$1" "$TMP/archived"; } | |
| is_gone() { grep -qx -- "$1" "$TMP/gone"; } | |
| # ---------------------------------------------------------------- phase 4: update | |
| note() { printf '%s\t%s\t%s\n' "$1" "$2" "$3" >> "$TMP/report"; } | |
| # true only if the clone holds nothing a human could miss | |
| holds_nothing() { | |
| d=$1 | |
| git -C "$d" rev-parse -q --verify HEAD >/dev/null 2>&1 && return 1 # has commits | |
| [ "$(ls -A "$d" | grep -vx '.git' | grep -c .)" -eq 0 ] || return 1 # has files | |
| [ "$(git -C "$d" stash list 2>/dev/null | grep -c .)" -eq 0 ] || return 1 | |
| [ "$(git -C "$d" worktree list 2>/dev/null | grep -c .)" -le 1 ] || return 1 | |
| [ "$(git -C "$d" for-each-ref refs/heads refs/tags 2>/dev/null | grep -c .)" -eq 0 ] || return 1 | |
| return 0 | |
| } | |
| # a repo deleted from GitHub: remove it only if it is a hollow shell | |
| handle_deleted() { | |
| d=$1 | |
| # paranoia: this is the only irreversible operation in the script | |
| case "$d" in ""|.|..|/*|*/*) note DELETED "$d" "refusing to prune a suspicious path"; return ;; esac | |
| [ -d "./$d/.git" ] || { note DELETED "$d" "refusing to prune: not a git clone"; return; } | |
| if ! holds_nothing "$d"; then | |
| note DELETED "$d" "gone from GitHub but has local content — untouched, may be the only copy"; return | |
| fi | |
| if [ "$PRUNE_DEAD" -eq 0 ]; then | |
| note DELETED-EMPTY "$d" "gone from GitHub, empty clone — kept (--no-prune-deleted)"; return | |
| fi | |
| if [ "$APPLY" -eq 0 ]; then | |
| note WOULD-REMOVE "$d" "gone from GitHub, no commits and no files — would delete the directory"; return | |
| fi | |
| if rm -rf "./$d"; then note REMOVED "$d" "gone from GitHub, empty clone — directory deleted" | |
| else note DELETED "$d" "gone from GitHub, empty clone — delete failed"; fi | |
| } | |
| # echoes "TAG<TAB>detail"; fast-forwards in place when that is safe | |
| classify() { | |
| d=$1 | |
| url=$(git -C "$d" remote get-url origin 2>/dev/null) | |
| [ -n "$url" ] || { printf 'NO-REMOTE\tno '\''origin'\'' remote — local-only repo'; return; } | |
| if fetch_failed "$d"; then printf 'UNREACHABLE\tfetch failed (%s) — state is stale, not verified' "$(fetch_error "$d")"; return; fi | |
| branch=$(git -C "$d" symbolic-ref --short -q HEAD) | |
| [ -n "$branch" ] || { printf 'DETACHED\tdetached HEAD — resolve by hand'; return; } | |
| # '@{u}' does not resolve on an unborn branch, so fall back to the config | |
| up=$(git -C "$d" rev-parse --abbrev-ref --symbolic-full-name '@{u}' 2>/dev/null) | |
| if [ -z "$up" ] || [ "$up" = '@{u}' ]; then | |
| ur=$(git -C "$d" config --get "branch.$branch.remote" 2>/dev/null) | |
| um=$(git -C "$d" config --get "branch.$branch.merge" 2>/dev/null) | |
| if [ -n "$ur" ] && [ -n "$um" ]; then up="$ur/${um#refs/heads/}"; else up=""; fi | |
| fi | |
| [ -n "$up" ] || { printf 'NO-UPSTREAM\tbranch '\''%s'\'' tracks nothing' "$branch"; return; } | |
| has_head=1; git -C "$d" rev-parse -q --verify HEAD >/dev/null 2>&1 || has_head=0 | |
| if ! git -C "$d" rev-parse -q --verify "$up^{commit}" >/dev/null 2>&1; then | |
| if [ "$has_head" -eq 0 ]; then printf 'EMPTY-BOTH\tno commits locally or on %s — nothing to sync' "$up" | |
| else printf 'UPSTREAM-GONE\t%s -> %s no longer exists on the remote' "$branch" "$up"; fi | |
| return | |
| fi | |
| # tracked modifications block us; untracked files do not (git refuses to | |
| # overwrite an untracked file during a merge, so it fails safe on its own) | |
| if [ -n "$(git -C "$d" status --porcelain --untracked-files=no 2>/dev/null)" ]; then | |
| printf 'DIRTY\tuncommitted changes to tracked files on '\''%s'\''' "$branch"; return | |
| fi | |
| if [ "$has_head" -eq 1 ]; then | |
| counts=$(git -C "$d" rev-list --left-right --count "$up...HEAD" 2>/dev/null) | |
| behind=${counts%%[!0-9]*}; ahead=${counts##*[!0-9]} | |
| if [ "${ahead:-0}" -gt 0 ] && [ "${behind:-0}" -gt 0 ]; then | |
| printf 'DIVERGED\t%s is %s ahead / %s behind %s — left alone' "$branch" "$ahead" "$behind" "$up"; return | |
| elif [ "${ahead:-0}" -gt 0 ]; then | |
| printf 'AHEAD\t%s has %s unpushed commit(s), nothing to pull' "$branch" "$ahead"; return | |
| elif [ "${behind:-0}" -eq 0 ]; then | |
| printf 'UP-TO-DATE\t%s' "$branch"; return | |
| fi | |
| action=FF; detail=$(printf '%s: %s commit(s) behind %s' "$branch" "$behind" "$up") | |
| else | |
| action=EMPTY-CLONE; detail=$(printf 'no commits — restores from %s' "$up") | |
| fi | |
| case "$branch" in main|master|develop|trunk) ;; *) detail="$detail [feature branch — default branch stays stale]" ;; esac | |
| if [ "$APPLY" -eq 0 ]; then printf 'WOULD-%s\t%s' "$action" "$detail" | |
| elif out=$(git -C "$d" merge --ff-only "$up" 2>&1); then printf '%s\t%s' "$action" "$detail" | |
| else printf 'FF-BLOCKED\t%s' "$(printf '%s' "$out" | head -1)"; fi | |
| } | |
| # archived repos: capture the final state, then get them out of the way | |
| retire() { | |
| d=$1; how=$2 | |
| if [ "$(git -C "$d" worktree list 2>/dev/null | grep -c .)" -gt 1 ]; then | |
| note ARCHIVE-SKIP "$d" "archived; $how — NOT moved: linked worktree(s) would break"; return | |
| fi | |
| if [ -e "$ARCHIVE_DIR/$d" ]; then | |
| note ARCHIVE-SKIP "$d" "archived; $how — NOT moved: $ARCHIVE_DIR/$d already exists"; return | |
| fi | |
| if [ "$APPLY" -eq 0 ]; then | |
| note WOULD-ARCHIVE "$d" "archived; $how — would move to $ARCHIVE_DIR/"; return | |
| fi | |
| mkdir -p "$ARCHIVE_DIR" | |
| if mv "$d" "$ARCHIVE_DIR/$d"; then note ARCHIVED-MOVED "$d" "archived; $how — moved to $ARCHIVE_DIR/" | |
| else note ARCHIVE-SKIP "$d" "archived; $how — mv failed"; fi | |
| } | |
| while IFS= read -r d; do | |
| if is_gone "$d"; then handle_deleted "$d"; continue; fi | |
| res=$(classify "$d") | |
| tag=${res%% *}; det=${res#* } | |
| if is_archived "$d"; then retire "$d" "$tag${det:+ ($det)}"; else note "$tag" "$d" "$det"; fi | |
| done < "$TMP/repos" | |
| while IFS=$'\t' read -r d real; do | |
| [ -n "$d" ] && note RENAMED "$d" "repo was renamed to '$real' — you may end up with both" | |
| done < "$TMP/renamed" | |
| while IFS=$'\t' read -r d url; do | |
| [ -n "$d" ] && note FOREIGN "$d" "belongs to another org: $url" | |
| done < "$TMP/foreign" | |
| # ---------------------------------------------------------------- report | |
| echo "== results" | |
| for tag in FF WOULD-FF EMPTY-CLONE WOULD-EMPTY-CLONE ARCHIVED-MOVED WOULD-ARCHIVE ARCHIVE-SKIP \ | |
| REMOVED WOULD-REMOVE DELETED-EMPTY DELETED FF-BLOCKED DIRTY DIVERGED AHEAD \ | |
| NO-UPSTREAM UPSTREAM-GONE EMPTY-BOTH DETACHED NO-REMOTE UNREACHABLE RENAMED FOREIGN; do | |
| n=$(awk -F'\t' -v t="$tag" '$1==t' "$TMP/report" | grep -c .) | |
| [ "$n" -eq 0 ] && continue | |
| echo; echo "-- $tag ($n)" | |
| awk -F'\t' -v t="$tag" '$1==t {printf " %-42s %s\n", $2, $3}' "$TMP/report" | |
| done | |
| echo; echo "-- summary" | |
| printf ' %s up to date, %s updated/updatable, %s retired, %s pruned, %s skipped, %s new to clone\n' \ | |
| "$(awk -F'\t' '$1=="UP-TO-DATE"' "$TMP/report" | grep -c .)" \ | |
| "$(awk -F'\t' '$1~/^(WOULD-)?(FF|EMPTY-CLONE)$/' "$TMP/report" | grep -c .)" \ | |
| "$(awk -F'\t' '$1~/^(ARCHIVED-MOVED|WOULD-ARCHIVE)$/' "$TMP/report" | grep -c .)" \ | |
| "$(awk -F'\t' '$1~/^(REMOVED|WOULD-REMOVE)$/' "$TMP/report" | grep -c .)" \ | |
| "$(awk -F'\t' '$1~/^(DIRTY|DIVERGED|AHEAD|NO-UPSTREAM|UPSTREAM-GONE|DETACHED|NO-REMOTE|FF-BLOCKED|UNREACHABLE|ARCHIVE-SKIP)$/' "$TMP/report" | grep -c .)" \ | |
| "$MISSING_N" | |
| if awk -F'\t' '$1=="DELETED"' "$TMP/report" | grep -q .; then | |
| echo " WARNING: $(awk -F'\t' '$1=="DELETED"' "$TMP/report" | grep -c .) repo(s) with local content no longer exist on GitHub — your clone may be the only copy." | |
| fi | |
| [ "$APPLY" -eq 0 ] && echo " (dry run — re-run with --apply)" | |
| exit 0 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment