gitlab clone all projects

gistfile1.sh

#!/usr/bin/env bash

# should install jq first, for mac:brew install jq

# Documentation
# https://docs.gitlab.com/ce/api/projects.html#list-projects

NAMESPACE=""
BASE_PATH="https://xxxxx/"
PROJECT_SEARCH_PARAM=""
PROJECT_SELECTION="select(.namespace.full_path | contains(\"$NAMESPACE\"))"
INCLUDE_SUBGROUPS="true"

#below use git@xxx to git clone
PROJECT_PROJECTION="{ "path": .path_with_namespace, "git": .ssh_url_to_repo }"
#below use http:// to git clone
#PROJECT_PROJECTION="{ "path": .path, "git": .http_url_to_repo }"

GITLAB_PRIVATE_TOKEN="xxxxx"

FILENAME="repos.json"

echo "Starting script execution..."
echo "Fetching projects from GitLab..."

trap "{ rm -f $FILENAME; }" EXIT

# Clear the file if it exists
> "$FILENAME"

# Get total number of pages from the first request
TOTAL_PAGES=$(curl -s "${BASE_PATH}api/v4/projects?private_token=$GITLAB_PRIVATE_TOKEN&search=$PROJECT_SEARCH_PARAM&include_subgroups=true&per_page=100" \
    -I | grep "x-total-pages:" | cut -d' ' -f2 | tr -d '\r')

echo "Total pages to fetch: $TOTAL_PAGES"

# Fetch all pages
for page in $(seq 1 $TOTAL_PAGES); do
    echo "Fetching page $page of $TOTAL_PAGES..."
    curl -s "${BASE_PATH}api/v4/projects?private_token=$GITLAB_PRIVATE_TOKEN&search=$PROJECT_SEARCH_PARAM&include_subgroups=true&per_page=100&page=$page" \
        | jq --raw-output --compact-output ".[] | $PROJECT_SELECTION | $PROJECT_PROJECTION" >> "$FILENAME"
done

echo "Checking if repos.json was created and has content..."
if [ -s "$FILENAME" ]; then
    echo "repos.json exists and has content"
    echo "Number of projects found: $(wc -l < "$FILENAME")"
    echo "Projects to be processed:"
    cat "$FILENAME"
else
    echo "repos.json is empty or does not exist"
fi
    
while read repo; do
    THEPATH=$(echo "$repo" | jq -r ".path")
    GIT=$(echo "$repo" | jq -r ".git")
    
    echo "Processing repository: $THEPATH"
    
    if [ ! -d "$THEPATH" ]; then
        echo "Creating directory structure for $THEPATH"
        mkdir -p "$THEPATH"
        echo "Cloning $THEPATH ( $GIT )"
        git clone "$GIT" "$THEPATH" &
    else
        echo "Pulling $THEPATH"
        (cd "$THEPATH" && git pull) &
    fi
done < "$FILENAME"

wait
echo "Script execution completed."

#! /bin/bash

GITLAB_HOST="xxx.xxx.cn"
GROUP_ID=7
PRIVATE_TOKEN=""
INCLUDE_SUBGROUPS=true

for repo in $(curl "https://${GITLAB_HOST}/api/v4/groups/${GROUP_ID}/projects?private_token=${PRIVATE_TOKEN}&include_subgroups=true" | jq .[].ssh_url_to_repo | tr -d '"'); 
  do git clone $repo; 
done;