LFS Migration by Object

lfs-migrate.sh

#!/bin/bash

# =============================================================================
# LFS Migration by Object ID
# =============================================================================
# script collects all unique LFS OIDs from the repository and pushes them
# directly by object ID
# =============================================================================

# Check bash version (requires bash 4+ for associative arrays)
if ((BASH_VERSINFO[0] < 4)); then
	echo "Error: This script requires bash version 4 or higher."
	echo "Current version: $BASH_VERSION"
	exit 1
fi

# Initialize variables
SOURCE_PAT=""
SOURCE_URL=""
TARGET_ORG=""
TARGET_REPO=""
GHEC_TOKEN=""
BATCH_SIZE=50
DRY_RUN=false

# Function to display usage
usage() {
	echo "Usage: $0 -s <source_pat> -u <source_repo_url> -o <target_org> -r <target_repo> -t <ghec_token> [-n <batch_size>] [-d]"
	echo ""
	echo "Options:"
	echo "  -s  Source repository PAT token for GHES"
	echo "  -u  Source repository URL"
	echo "  -o  Target organization in GHEC"
	echo "  -r  Target repository name in GHEC"
	echo "  -t  GHEC PAT token for push"
	echo "  -n  Number of OIDs per batch (default: 50)"
	echo "  -d  Dry run — show what would be pushed without pushing"
	echo "  -h  Show this help message"
	echo ""
	echo "This script pushes LFS objects by OID instead of by branch."
	echo "It collects all unique OIDs across all branches and pushes them"
	echo "in batches, which is significantly faster for repositories with"
	echo "many branches sharing the same LFS objects."
	exit 1
}

# Format bytes into human-readable size
format_size() {
	local bytes=$1
	if [ "$bytes" -ge 1073741824 ]; then
		echo "$(awk "BEGIN {printf \"%.2f\", $bytes/1073741824}") GB"
	elif [ "$bytes" -ge 1048576 ]; then
		echo "$(awk "BEGIN {printf \"%.2f\", $bytes/1048576}") MB"
	elif [ "$bytes" -ge 1024 ]; then
		echo "$(awk "BEGIN {printf \"%.2f\", $bytes/1024}") KB"
	else
		echo "${bytes} B"
	fi
}

# Parse command line arguments
while getopts "s:u:o:r:t:n:dh" opt; do
	case $opt in
	s) SOURCE_PAT="$OPTARG" ;;
	u) SOURCE_URL="$OPTARG" ;;
	o) TARGET_ORG="$OPTARG" ;;
	r) TARGET_REPO="$OPTARG" ;;
	t) GHEC_TOKEN="$OPTARG" ;;
	n) BATCH_SIZE="$OPTARG" ;;
	d) DRY_RUN=true ;;
	h) usage ;;
	\?)
		echo "Invalid option -$OPTARG" >&2
		usage
		;;
	esac
done

# Validate required parameters
if [ -z "$SOURCE_PAT" ] || [ -z "$SOURCE_URL" ] || [ -z "$TARGET_ORG" ] || [ -z "$TARGET_REPO" ] || [ -z "$GHEC_TOKEN" ]; then
	echo "Error: All required parameters must be provided"
	usage
fi

# Validate batch size
if ! [[ "$BATCH_SIZE" =~ ^[0-9]+$ ]] || [ "$BATCH_SIZE" -lt 1 ]; then
	echo "Error: Batch size must be a positive integer"
	exit 1
fi

# Extract repository name from SOURCE_URL for the clone directory
SOURCE_REPO_NAME=$(basename "${SOURCE_URL%.git}")
CLONE_DIR="./${SOURCE_REPO_NAME}"

# Log files (absolute paths so they work before and after cd)
SCRIPT_DIR="$(pwd)"
LFS_MIGRATION_LOG="${SCRIPT_DIR}/lfs-migration-byobject.log"
LFS_OIDS_ALL_LOG="${SCRIPT_DIR}/lfs-oids-all-byobject.log"
LFS_PUSH_PROGRESS_LOG="${SCRIPT_DIR}/lfs-push-progress.log"
LFS_FAILED_OIDS_LOG="${SCRIPT_DIR}/lfs-failed-oids.log"

# Initialize log files (preserve progress log for resume support)
true >"$LFS_MIGRATION_LOG"
true >"$LFS_OIDS_ALL_LOG"
true >"$LFS_FAILED_OIDS_LOG"

log() {
	local msg
	msg="[$(date '+%Y-%m-%d %H:%M:%S')] $1"
	echo "$msg"
	echo "$msg" >>"$LFS_MIGRATION_LOG"
}

log "========================================="
log "LFS Migration by Object ID - Started"
log "========================================="
log "Source URL:   $SOURCE_URL"
log "Target:       ${TARGET_ORG}/${TARGET_REPO}"
log "Batch size:   $BATCH_SIZE"
log "Dry run:      $DRY_RUN"
log ""

# Clone source repository if it doesn't exist, otherwise reuse
if [ -d "$CLONE_DIR" ]; then
	log "Reusing existing repository clone at $CLONE_DIR"
else
	log "Cloning source repository (mirror) to $CLONE_DIR..."
	mkdir -p "$CLONE_DIR"
	if ! git clone --mirror "https://x-access-token:${SOURCE_PAT}@${SOURCE_URL#https://}" "$CLONE_DIR" 2>&1 | tee -a "$LFS_MIGRATION_LOG"; then
		log "Error: Failed to clone source repository"
		exit 1
	fi
fi

# Fetch all LFS objects from source
log "Fetching all LFS objects from source..."
if ! git -C "$CLONE_DIR" lfs fetch --all 2>&1 | tee -a "$LFS_MIGRATION_LOG"; then
	log "Warning: LFS fetch encountered errors (some objects may be missing)"
fi

# Change directory
cd "$CLONE_DIR" || exit 1

# Gather repository context (branch count, total LFS references)
log ""
log "========================================="
log "LFS Repository Summary"
log "========================================="

TOTAL_BRANCHES=$(git for-each-ref --format='%(refname)' refs/heads/ | wc -l | tr -d ' ')
log "Total branches in repo:  $TOTAL_BRANCHES"

# Discover unique OIDs using git lfs ls-files -l --all (reliable across git-lfs versions)
# Format: "<64char_oid> * <filename>" or "<64char_oid> - <filename>"
declare -A OID_SIZE_MAP
declare -A SEEN_OIDS
ALL_UNIQUE_OIDS=()

LFS_LS_OUTPUT=$(git lfs ls-files -l --all 2>/dev/null)

# Count total LFS references (non-unique, across all branches) for context
TOTAL_LFS_REFERENCES=$(echo "$LFS_LS_OUTPUT" | grep -c . || echo 0)
log "Total LFS references:    $TOTAL_LFS_REFERENCES (across all branches)"

# Extract unique OIDs
while IFS= read -r line; do
	[ -z "$line" ] && continue
	oid=$(echo "$line" | awk '{print $1}')
	# Validate it looks like a sha256 OID
	if [[ "$oid" =~ ^[a-f0-9]{64}$ ]] && [ -z "${SEEN_OIDS[$oid]+x}" ]; then
		SEEN_OIDS["$oid"]=1
		ALL_UNIQUE_OIDS+=("$oid")
	fi
done <<<"$LFS_LS_OUTPUT"

TOTAL_UNIQUE_OIDS=${#ALL_UNIQUE_OIDS[@]}

# Get sizes by checking the local LFS object store (populated by git lfs fetch --all)
# In a bare/mirror repo, objects are at: lfs/objects/<oid[0:2]>/<oid[2:4]>/<oid>
# In a regular repo, objects are at: .git/lfs/objects/<oid[0:2]>/<oid[2:4]>/<oid>
TOTAL_LFS_SIZE_BYTES=0
if [ -d "lfs/objects" ]; then
	LFS_OBJECTS_DIR="lfs/objects"
elif [ -d ".git/lfs/objects" ]; then
	LFS_OBJECTS_DIR=".git/lfs/objects"
else
	LFS_OBJECTS_DIR=""
fi

for oid in "${ALL_UNIQUE_OIDS[@]}"; do
	obj_size=0
	if [ -n "$LFS_OBJECTS_DIR" ]; then
		obj_path="${LFS_OBJECTS_DIR}/${oid:0:2}/${oid:2:2}/${oid}"
		if [ -f "$obj_path" ]; then
			obj_size=$(wc -c <"$obj_path" | tr -d ' ')
		fi
	fi
	OID_SIZE_MAP["$oid"]=$obj_size
	TOTAL_LFS_SIZE_BYTES=$((TOTAL_LFS_SIZE_BYTES + obj_size))
done

if [ "$TOTAL_UNIQUE_OIDS" -eq 0 ]; then
	log "No LFS objects found in repository. Nothing to migrate."
	cd ..
	exit 0
fi

# Write all OIDs to log
printf '%s\n' "${ALL_UNIQUE_OIDS[@]}" >"$LFS_OIDS_ALL_LOG"

log "Unique LFS objects:      $TOTAL_UNIQUE_OIDS"
log "Total unique size:       $(format_size "$TOTAL_LFS_SIZE_BYTES")"
log "========================================="
log ""

# Setup target repo URL and authentication
GH_REPO_CLONE_URL="https://github.com/${TARGET_ORG}/${TARGET_REPO}.git"
AUTH=$(echo -n "x-access-token:${GHEC_TOKEN}" | openssl base64 | tr -d '\n')

# Configure git
git config user.name github-actions
git config user.email github-actions@github.com

# Add or update the new remote
if git remote | grep -q "^new$"; then
	git remote set-url new "$GH_REPO_CLONE_URL"
else
	git remote add new "$GH_REPO_CLONE_URL"
fi

# Test authentication to target repository
log "Testing authentication to target repository..."
if ! git -c "http.${GH_REPO_CLONE_URL}.extraheader=Authorization: Basic $AUTH" \
	ls-remote "$GH_REPO_CLONE_URL" HEAD &>/dev/null; then
	log "Error: Cannot authenticate to target repository. Check your GHEC token."
	cd ..
	exit 1
fi
log "✓ Authentication successful"
log ""

# Check for previously pushed OIDs (resume support)
declare -A PUSHED_OIDS
ALREADY_PUSHED=0
if [ -f "$LFS_PUSH_PROGRESS_LOG" ] && [ -s "$LFS_PUSH_PROGRESS_LOG" ]; then
	while IFS= read -r pushed_oid; do
		[ -z "$pushed_oid" ] && continue
		PUSHED_OIDS["$pushed_oid"]=1
		ALREADY_PUSHED=$((ALREADY_PUSHED + 1))
	done <"$LFS_PUSH_PROGRESS_LOG"
	if [ "$ALREADY_PUSHED" -gt 0 ]; then
		log "Resuming: Found $ALREADY_PUSHED previously pushed OIDs"
	fi
fi

# Build list of OIDs still needing to be pushed
OIDS_TO_PUSH=()
OIDS_TO_PUSH_SIZE=0
for oid in "${ALL_UNIQUE_OIDS[@]}"; do
	if [ -z "${PUSHED_OIDS[$oid]+x}" ]; then
		OIDS_TO_PUSH+=("$oid")
		OIDS_TO_PUSH_SIZE=$((OIDS_TO_PUSH_SIZE + ${OID_SIZE_MAP[$oid]:-0}))
	fi
done

REMAINING_OIDS=${#OIDS_TO_PUSH[@]}

if [ "$REMAINING_OIDS" -eq 0 ]; then
	log "All $TOTAL_UNIQUE_OIDS OIDs have already been pushed. Nothing to do."
	cd ..
	exit 0
fi

log "OIDs to push:  $REMAINING_OIDS ($(format_size "$OIDS_TO_PUSH_SIZE"))"
if [ "$ALREADY_PUSHED" -gt 0 ]; then
	log "Already pushed: $ALREADY_PUSHED"
fi
log ""

if [ "$DRY_RUN" = true ]; then
	log "========================================="
	log "DRY RUN — Objects that would be pushed:"
	log "========================================="
	for oid in "${OIDS_TO_PUSH[@]}"; do
		log "  $oid  $(format_size "${OID_SIZE_MAP[$oid]:-0}")"
	done
	log ""
	log "Total: $REMAINING_OIDS objects, $(format_size "$OIDS_TO_PUSH_SIZE")"
	log "========================================="
	cd ..
	exit 0
fi

# Push OIDs in batches
TOTAL_BATCHES=$(((REMAINING_OIDS + BATCH_SIZE - 1) / BATCH_SIZE))
BATCH_NUM=0
PUSH_SUCCESS=0
PUSH_FAILED=0
PUSH_SUCCESS_SIZE=0
PUSH_FAILED_SIZE=0
PUSH_START_TIME=$(date +%s)

log "========================================="
log "Pushing LFS objects in $TOTAL_BATCHES batches of up to $BATCH_SIZE OIDs"
log "========================================="
log ""

for ((i = 0; i < REMAINING_OIDS; i += BATCH_SIZE)); do
	BATCH_NUM=$((BATCH_NUM + 1))

	# Slice the batch
	BATCH_OIDS=("${OIDS_TO_PUSH[@]:$i:$BATCH_SIZE}")
	BATCH_COUNT=${#BATCH_OIDS[@]}

	# Calculate batch size
	BATCH_SIZE_BYTES=0
	for oid in "${BATCH_OIDS[@]}"; do
		BATCH_SIZE_BYTES=$((BATCH_SIZE_BYTES + ${OID_SIZE_MAP[$oid]:-0}))
	done

	# Elapsed time and ETA
	CURRENT_TIME=$(date +%s)
	ELAPSED=$((CURRENT_TIME - PUSH_START_TIME))
	OIDS_DONE=$((i))
	if [ "$OIDS_DONE" -gt 0 ]; then
		SECS_PER_OID=$(awk "BEGIN {printf \"%.2f\", $ELAPSED / $OIDS_DONE}")
		OIDS_LEFT=$((REMAINING_OIDS - OIDS_DONE))
		ETA_SECS=$(awk "BEGIN {printf \"%.0f\", $SECS_PER_OID * $OIDS_LEFT}")
		ETA_MIN=$((ETA_SECS / 60))
		ETA_HR=$((ETA_MIN / 60))
		ETA_MIN_REM=$((ETA_MIN % 60))
		ETA_DISPLAY="${ETA_HR}h ${ETA_MIN_REM}m"
	else
		ETA_DISPLAY="calculating..."
	fi

	log "Batch $BATCH_NUM/$TOTAL_BATCHES: $BATCH_COUNT OIDs ($(format_size "$BATCH_SIZE_BYTES")) [ETA: $ETA_DISPLAY]"

	# Push the batch using --object-id with stdin
	BATCH_PUSH_OUTPUT=$(printf '%s\n' "${BATCH_OIDS[@]}" |
		git -c "http.${GH_REPO_CLONE_URL}.extraheader=Authorization: Basic $AUTH" \
			lfs push --object-id new --stdin 2>&1)
	BATCH_EXIT_CODE=$?

	if [ $BATCH_EXIT_CODE -eq 0 ]; then
		log "  ✓ Batch $BATCH_NUM succeeded ($BATCH_COUNT objects, $(format_size "$BATCH_SIZE_BYTES"))"
		PUSH_SUCCESS=$((PUSH_SUCCESS + BATCH_COUNT))
		PUSH_SUCCESS_SIZE=$((PUSH_SUCCESS_SIZE + BATCH_SIZE_BYTES))

		# Record pushed OIDs for resume support
		printf '%s\n' "${BATCH_OIDS[@]}" >>"$LFS_PUSH_PROGRESS_LOG"
	else
		log "  ✗ Batch $BATCH_NUM failed (exit code $BATCH_EXIT_CODE)"
		log "    Output: $BATCH_PUSH_OUTPUT"

		# Fall back to pushing OIDs individually to identify which ones failed
		log "  Retrying individually..."
		for oid in "${BATCH_OIDS[@]}"; do
			OID_SIZE=${OID_SIZE_MAP[$oid]:-0}
			INDIVIDUAL_OUTPUT=$(echo "$oid" |
				git -c "http.${GH_REPO_CLONE_URL}.extraheader=Authorization: Basic $AUTH" \
					lfs push --object-id new --stdin 2>&1)
			INDIVIDUAL_EXIT=$?

			if [ $INDIVIDUAL_EXIT -eq 0 ]; then
				PUSH_SUCCESS=$((PUSH_SUCCESS + 1))
				PUSH_SUCCESS_SIZE=$((PUSH_SUCCESS_SIZE + OID_SIZE))
				echo "$oid" >>"$LFS_PUSH_PROGRESS_LOG"
				log "    ✓ $oid ($(format_size "$OID_SIZE"))"
			else
				PUSH_FAILED=$((PUSH_FAILED + 1))
				PUSH_FAILED_SIZE=$((PUSH_FAILED_SIZE + OID_SIZE))
				echo "$oid" >>"$LFS_FAILED_OIDS_LOG"
				log "    ✗ $oid ($(format_size "$OID_SIZE")): $INDIVIDUAL_OUTPUT"
			fi
		done
	fi
done

# Return to parent directory
cd ..

# Final timing
PUSH_END_TIME=$(date +%s)
TOTAL_ELAPSED=$((PUSH_END_TIME - PUSH_START_TIME))
TOTAL_ELAPSED_MIN=$((TOTAL_ELAPSED / 60))
TOTAL_ELAPSED_HR=$((TOTAL_ELAPSED_MIN / 60))
TOTAL_ELAPSED_MIN_REM=$((TOTAL_ELAPSED_MIN % 60))
TOTAL_ELAPSED_SEC=$((TOTAL_ELAPSED % 60))

# Print summary
log ""
log "========================================="
log "LFS Migration by Object ID — Summary"
log "========================================="
log ""
log "Repository:"
log "  Total branches:            $TOTAL_BRANCHES"
log "  Total LFS references:      $TOTAL_LFS_REFERENCES (across all branches)"
log "  Unique LFS objects:        $TOTAL_UNIQUE_OIDS"
log "  Total unique size:         $(format_size "$TOTAL_LFS_SIZE_BYTES")"
log ""
log "Push Results:"
log "  Previously pushed (resume): $ALREADY_PUSHED"
log "  Pushed this run:            $PUSH_SUCCESS ($(format_size "$PUSH_SUCCESS_SIZE"))"
log "  Failed this run:            $PUSH_FAILED ($(format_size "$PUSH_FAILED_SIZE"))"
log ""
log "Timing:"
log "  Total elapsed time:         ${TOTAL_ELAPSED_HR}h ${TOTAL_ELAPSED_MIN_REM}m ${TOTAL_ELAPSED_SEC}s"
if [ "$PUSH_SUCCESS" -gt 0 ]; then
	AVG_SECS=$(awk "BEGIN {printf \"%.1f\", $TOTAL_ELAPSED / $PUSH_SUCCESS}")
	log "  Avg time per object:        ${AVG_SECS}s"
fi
log ""
log "Log files:"
log "  - $LFS_MIGRATION_LOG      : Full migration log"
log "  - $LFS_OIDS_ALL_LOG        : All unique OIDs in the repo"
log "  - $LFS_PUSH_PROGRESS_LOG   : Successfully pushed OIDs (for resume)"
log "  - $LFS_FAILED_OIDS_LOG     : Failed OIDs (for retry)"
log "========================================="

if [ "$PUSH_FAILED" -gt 0 ]; then
	log ""
	log "WARNING: $PUSH_FAILED objects failed to push."
	log "Re-run this script to retry — it will skip already-pushed objects."
	exit 1
fi

log ""
log "✓ LFS migration by object ID completed successfully!"
log "Note: $CLONE_DIR has been preserved for reuse. Delete it manually if no longer needed."