gitlab-pipeline-for-parallel-sync-external-repos-for-migration

.gitlab-ci.yml

image: $GIT_SYNC_IMAGE

stages: [sync]

sync-file-based:
  stage: sync
  variables:
    GIT_SYNC_TARGET_PROJECT_BASE: "our.git.server.de/some/path/studydrive"
    SOURCE_TOKEN_VARIABLE_NAME: "GIT_SYNC_GENERIC_BITBUCKET_TOKEN"
    SOURCE_DEFAULT_BRANCH: "master"
  parallel:
    matrix:
      - REPO_NAME: "fun-defective"
  cache:
    key: "git-sync-file-based"
    paths:
      - work/
  script: |
    set -vex

    # require tokens
    GIT_SYNC_BITBUCKET_TOKEN="${!SOURCE_TOKEN_VARIABLE_NAME}"
    test -n "$GIT_SYNC_BITBUCKET_TOKEN"
    test -n "$GIT_SYNC_TARGET_TOKEN"
    test -n "$GIT_SYNC_TARGET_PROJECT_BASE"

    mkdir -p work
    cd work

    SOURCE_REPO_URL="https://johndoe:${GIT_SYNC_BITBUCKET_TOKEN}@bitbucket.org/StudyDrive/${REPO_NAME}.git"
    TARGET_URL="https://oauth2:${GIT_SYNC_TARGET_TOKEN}@${GIT_SYNC_TARGET_PROJECT_BASE}/${REPO_NAME}.git"

    # clone or update source repository (read-only, for file extraction)
    if [ -d "${REPO_NAME}-source" ]; then
      cd "${REPO_NAME}-source"
      git fetch origin "${SOURCE_DEFAULT_BRANCH}" --depth=1 || true
      git checkout "origin/${SOURCE_DEFAULT_BRANCH}" || git checkout "${SOURCE_DEFAULT_BRANCH}"
      cd ..
    else
      git clone --depth=1 --branch="${SOURCE_DEFAULT_BRANCH}" "$SOURCE_REPO_URL" "${REPO_NAME}-source" || true
    fi

    # create or update target repository (clean, file-based sync)
    if [ ! -d "${REPO_NAME}-target" ]; then
      echo "First run: creating new clean repository..."
      mkdir "${REPO_NAME}-target"
      cd "${REPO_NAME}-target"
      git init
      git remote add origin "$TARGET_URL"
      
      # try to fetch existing repo from gitlab
      git fetch origin "${SOURCE_DEFAULT_BRANCH}" || true
      if git rev-parse "origin/${SOURCE_DEFAULT_BRANCH}" >/dev/null 2>&1; then
        git checkout -b "${SOURCE_DEFAULT_BRANCH}" "origin/${SOURCE_DEFAULT_BRANCH}"
      else
        git checkout -b "${SOURCE_DEFAULT_BRANCH}"
      fi
      cd ..
    fi

    cd "${REPO_NAME}-target"
    git checkout "${SOURCE_DEFAULT_BRANCH}" || git checkout -b "${SOURCE_DEFAULT_BRANCH}"

    # rsync files from source to target (excluding .git)
    echo "Syncing files from source to target..."
    rsync -av --delete --exclude='.git' "../${REPO_NAME}-source/" ./

    # check if there are changes
    if git diff --quiet && git diff --cached --quiet; then
      echo "No changes detected, nothing to sync"
      exit 0
    fi

    # commit and push changes
    git add -A
    COMMIT_MSG="Sync from Bitbucket ${SOURCE_DEFAULT_BRANCH} at $(date -u +%Y-%m-%dT%H:%M:%SZ)"
    git commit -m "$COMMIT_MSG" || true
    
    echo "Pushing to GitLab..."
    git push origin "${SOURCE_DEFAULT_BRANCH}"

    echo "=== FILE-BASED SYNC COMPLETE ==="

sync-from-bitbucket:
  stage: sync
  variables:
    GIT_SYNC_TRACK_BRANCH: upstream-main        # target branch that always equals source default branch
    GIT_SYNC_UPSTREAM_MAIN: release                # fallback if source default branch cannot be detected
    GIT_SYNC_SOURCE_URL: "https"               # required, see matrix below
    GIT_SYNC_SOURCE_URL_BASE: "bitbucket.org/StudyDrive"               # optional, not used for Bitbucket
    GIT_SYNC_TARGET_PROJECT_BASE: "our.git.server.de/some/path/studydrive"
    SOURCE_TOKEN_VARIABLE_NAME: "GIT_SYNC_GENERIC_BITBUCKET_TOKEN" # token variable name for Bitbucket
    KUBERNETES_MEMORY_REQUEST: "512Mi"
    KUBERNETES_MEMORY_LIMIT: "768Mi"
    KUBERNETES_CPU_REQUEST: "1500m"
    KUBERNETES_CPU_LIMIT: "3000m"
  cache:
    key: "git-sync"
    paths:
      - work/
  parallel:
    matrix:
      - REPO_NAME: "fun-api"
      - REPO_NAME: "fun-web"
      - REPO_NAME: "fun-mobile"
      - REPO_NAME: "fun-backend"
      - REPO_NAME: "fun-auth"
      - REPO_NAME: "fun-processor"
      - REPO_NAME: "fun-analytics"
      - REPO_NAME: "fun-data"
      - REPO_NAME: "fun-service-a"
      - REPO_NAME: "fun-service-b"
      - REPO_NAME: "fun-service-c"
      - REPO_NAME: "fun-docker-base"
      - REPO_NAME: "fun-documents"
      - REPO_NAME: "fun-rewards"
      - REPO_NAME: "fun-communities"
      - REPO_NAME: "fun-targeting"
      - REPO_NAME: "fun-infrastructure"
      - REPO_NAME: "fun-docs"
      - REPO_NAME: "fun-local"
      - REPO_NAME: "fun-search"
      - REPO_NAME: "fun-messaging"
      - REPO_NAME: "fun-notifications"
      - REPO_NAME: "fun-jobs"
      - REPO_NAME: "fun-billing"
      - REPO_NAME: "fun-client"
      - REPO_NAME: "fun-search-v2"
      - REPO_NAME: "fun-storage"
      - REPO_NAME: "fun-dropdown"
      - REPO_NAME: "fun-streaming"
      - REPO_NAME: "fun-android"
      - REPO_NAME: "fun-images"
      - REPO_NAME: "fun-tags"
      - REPO_NAME: "fun-e2e"
      - REPO_NAME: "fun-ci"
      - REPO_NAME: "fun-ci-docker"
      - REPO_NAME: "fun-uploader"
      - REPO_NAME: "fun-vector-search"
      - REPO_NAME: "fun-frontend"
      - REPO_NAME: "fun-frontend-test"
      - REPO_NAME: "fun-processor-v2"
  script: |
    set -vex

    # require source token and target
    GIT_SYNC_BITBUCKET_TOKEN="${!SOURCE_TOKEN_VARIABLE_NAME}"
    DEFAULT_BRANCH="${SOURCE_DEFAULT_BRANCH:-$GIT_SYNC_UPSTREAM_MAIN}"

    test -n "$GIT_SYNC_BITBUCKET_TOKEN"
    test -n "$GIT_SYNC_TARGET_TOKEN"
    test -n "$GIT_SYNC_TARGET_PROJECT_BASE"

    # clone or update source repository
    mkdir -p "work"
    cd work
    SOURCE_REPO_URL="https://johndoe:${GIT_SYNC_BITBUCKET_TOKEN}@bitbucket.org/StudyDrive/${REPO_NAME}.git"
    
    if [ -d "${REPO_NAME}" ]; then
      cd "${REPO_NAME}"
      git remote set-url origin "$SOURCE_REPO_URL"
      
      # always fetch from source to get new changes
      git fetch --all --prune
    else
      git clone --mirror "$SOURCE_REPO_URL" "${REPO_NAME}/.git"
      cd "${REPO_NAME}"
      git config --unset core.bare
      git config core.logallrefupdates true
      git reset --hard
    fi

    # add or update target as push remote
    TARGET_URL="https://oauth2:${GIT_SYNC_TARGET_TOKEN}@${GIT_SYNC_TARGET_PROJECT_BASE}/${REPO_NAME}.git"
    if git remote | grep -q '^gitlab-remote$'; then
      git remote set-url gitlab-remote "$TARGET_URL"
    else
      git remote add gitlab-remote "$TARGET_URL"
    fi

    # detect source default branch, fallback to GIT_SYNC_UPSTREAM_MAIN
    UPSTREAM_MAIN="${SOURCE_DEFAULT_BRANCH:-$GIT_SYNC_UPSTREAM_MAIN}"

    # create tracking branch that mirrors source default
    git branch -f "$GIT_SYNC_TRACK_BRANCH" "$UPSTREAM_MAIN"

    # push all branches and tags to target (non-mirror push preserves extra branches in target)
    git push gitlab-remote --force 'refs/heads/*:refs/heads/*'
    git push gitlab-remote --force 'refs/tags/*:refs/tags/*'