gitlab-upgrade-helper

main.go

package main

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"strings"
)

var (
	gitlabInstance = getEnv("GITLAB_INSTANCE", "gitlab.example.com/upgradechecks/gitlab")
)

func main() {
	// Check if GitLab is installed via Docker
	if isDockerInstalled() && isGitLabRunningInDocker() {
		fmt.Println("Detected GitLab installation via Docker.")
		preUpgradeChecks()
		checkBackgroundMigrationsDocker()
		performBackupDocker()
		upgradeGitLabDocker()
		postUpgradeChecks()
	} else {
		fmt.Println("Detected GitLab installation via Omnibus.")
		preUpgradeChecks()
		checkBackgroundMigrationsOmnibus()
		performBackupOmnibus()
		upgradeGitLabOmnibus()
		postUpgradeChecks()
	}
}

// Check if Docker is installed
func isDockerInstalled() bool {
	cmd := exec.Command("docker", "ps")
	output, err := cmd.CombinedOutput()
	fmt.Println(string(output))
	if err != nil {
		fmt.Println("Docker is not installed.")
		return false
	}
	return true
}

// Check if GitLab is running in Docker container
func isGitLabRunningInDocker() bool {
	cmd := exec.Command("docker", "ps", "-q", "--filter", "name=gitlab")
	output, err := cmd.CombinedOutput()
	fmt.Println(string(output))
	if err != nil || len(output) == 0 {
		fmt.Println("GitLab is not running in a Docker container.")
		return false
	}
	return true
}

// Perform pre-upgrade checks
func preUpgradeChecks() {
	fmt.Println("Performing pre-upgrade checks...")
	// Clone repo via SSH
	cloneRepoCmd := exec.Command("git", "clone", "[email protected]:"+gitlabInstance+".git")
	cloneRepoCmd.Dir = os.Getenv("HOME")
	output, err := cloneRepoCmd.CombinedOutput()
	fmt.Println(string(output))
	if err != nil {
		log.Fatal("Failed to clone repository:", err)
	}

	// Update file
	updateFileCmd := exec.Command("sh", "-c", "echo 'Date: $(date)\nSystem Info: $(uname -a)\nErrors: $(dmesg)' >> test.txt")
	updateFileCmd.Dir = os.Getenv("HOME") + "/" + gitlabInstance
	output, err = updateFileCmd.CombinedOutput()
	fmt.Println(string(output))
	if err != nil {
		log.Fatal("Failed to update file:", err)
	}

	// Push changes
	pushChangesCmd := exec.Command("git", "add", "test.txt")
	pushChangesCmd.Dir = os.Getenv("HOME") + "/" + gitlabInstance
	output, err = pushChangesCmd.CombinedOutput()
	fmt.Println(string(output))
	if err != nil {
		log.Fatal("Failed to add file to repository:", err)
	}

	pushChangesCmd = exec.Command("git", "commit", "-m", "Updated file")
	pushChangesCmd.Dir = os.Getenv("HOME") + "/" + gitlabInstance
	output, err = pushChangesCmd.CombinedOutput()
	fmt.Println(string(output))
	if err != nil {
		log.Fatal("Failed to commit changes:", err)
	}

	pushChangesCmd = exec.Command("git", "push", "origin", "master")
	pushChangesCmd.Dir = os.Getenv("HOME") + "/" + gitlabInstance
	output, err = pushChangesCmd.CombinedOutput()
	fmt.Println(string(output))
	if err != nil {
		log.Fatal("Failed to push changes:", err)
	}

	// Delete the repository
	deleteRepoCmd := exec.Command("rm", "-rf", os.Getenv("HOME")+"/"+gitlabInstance)
	output, err = deleteRepoCmd.CombinedOutput()
	fmt.Println(string(output))
	if err != nil {
		log.Fatal("Failed to delete repository:", err)
	}

	fmt.Println("Pre-upgrade checks completed successfully.")
}

// Check background migrations for Omnibus install
func checkBackgroundMigrationsOmnibus() {
	fmt.Println("Checking background migrations...")
	cmd := exec.Command("gitlab-rails", "runner", "-e", "production", "puts Gitlab::BackgroundMigration.remaining")
	output, err := cmd.CombinedOutput()
	fmt.Println(string(output))
	if err != nil {
		log.Fatal("Failed to check background migrations:", err)
	}

	cmd = exec.Command("gitlab-rails", "runner", "-e", "production", "puts Gitlab::Database::BackgroundMigration::BatchedMigration.queued.count")
	output, err = cmd.CombinedOutput()
	fmt.Println(string(output))
	if err != nil {
		log.Fatal("Failed to check batched background migrations:", err)
	}
}

// Check background migrations for Docker install
func checkBackgroundMigrationsDocker() {
	fmt.Println("Checking background migrations...")
	cmd := exec.Command("docker", "exec", "-t", "gitlab", "gitlab-rails", "runner", "-e", "production", "puts Gitlab::BackgroundMigration.remaining")
	output, err := cmd.CombinedOutput()
	fmt.Println(string(output))
	if err != nil {
		log.Fatal("Failed to check background migrations:", err)
	}

	cmd = exec.Command("docker", "exec", "-t", "gitlab", "gitlab-rails", "runner", "-e", "production", "puts Gitlab::Database::BackgroundMigration::BatchedMigration.queued.count")
	output, err = cmd.CombinedOutput()
	fmt.Println(string(output))
	if err != nil {
		log.Fatal("Failed to check batched background migrations:", err)
	}
}

// Perform complete backup for Omnibus install
func performBackupOmnibus() {
	fmt.Println("Performing backup...")
	cmd := exec.Command("gitlab-backup", "create")
	output, err := cmd.CombinedOutput()
	fmt.Println(string(output))
	if err != nil {
		log.Fatal("Failed to perform backup:", err)
	}
}

// Perform complete backup for Docker install
func performBackupDocker() {
	fmt.Println("Performing backup...")
	cmd := exec.Command("docker", "exec", "-t", "gitlab", "gitlab-backup", "create")
	output, err := cmd.CombinedOutput()
	fmt.Println(string(output))
	if err != nil {
		log.Fatal("Failed to perform backup:", err)
	}
}

// Upgrade GitLab for Omnibus install
func upgradeGitLabOmnibus() {
	fmt.Println("Upgrading GitLab...")
	cmd := exec.Command("dnf", "install", "-y", "gitlab")
	output, err := cmd.CombinedOutput()
	fmt.Println(string(output))
	if err != nil {
		log.Fatal("Failed to upgrade GitLab:", err)
	}
}

// Upgrade GitLab for Docker install
func upgradeGitLabDocker() {
	fmt.Println("Upgrading GitLab...")
	workingDirCmd := exec.Command("docker", "inspect", "gitlab", "--format", "'{{index .Config.Labels \"com.docker.compose.project.working_dir\"}}'")
	output, err := workingDirCmd.CombinedOutput()
	fmt.Println(string(output))
	if err != nil {
		log.Fatal("Failed to get GitLab's working directory:", err)
	}
	workingDir := strings.Trim(string(output), "'\n")

	if workingDir == "" {
		log.Fatal("Failed to determine GitLab's working directory.")
	}

	cmd := exec.Command("docker-compose", "-f", workingDir+"/docker-compose.yml", "pull")
	output, err = cmd.CombinedOutput()
	fmt.Println(string(output))
	if err != nil {
		log.Fatal("Failed to pull GitLab Docker images:", err)
	}

	cmd = exec.Command("docker-compose", "-f", workingDir+"/docker-compose.yml", "up", "-d")
	output, err = cmd.CombinedOutput()
	fmt.Println(string(output))
	if err != nil {
		log.Fatal("Failed to start GitLab Docker containers:", err)
	}
}

// Perform post-upgrade checks
func postUpgradeChecks() {
	fmt.Println("Performing post-upgrade checks...")
	preUpgradeChecks()
	fmt.Println("Post-upgrade checks completed successfully.")
}

// Helper function to get environment variable value with fallback
func getEnv(key, fallback string) string {
	if value, ok := os.LookupEnv(key); ok {
		return value
	}
	return fallback
}

Prompt:

Write a Golang script that handle Gitlab Upgrades. 
Whenever running shell commands, print afterwards the output of the command.
Gitlab can be installed in 2 ways, via Docker and as a package.
To decide we try to run 'docker ps' and check if we can find a container running gitlab. when this fails or docker is not installed we can expect that we have a Omnibus install.  

The script checks first if the running instance is healthy. We call this pre upgrade checks. 
The pre upgrade checks includes cloning a repo via ssh with an key in the users Home Directory, update a file (printing the date, system infos errors of dmesg), pushing the changes. When this worked, delete the repo. 
The instance and name of the repo should be set via enviroment variables andd default to gitlab.example.com/upgradechecks/gitlab


Before starting the upgrade monitor if any background migrations still running as well as Batched background migrations. 
The commands to do so are:
  Omnibus install
    gitlab-rails runner -e production 'puts Gitlab::BackgroundMigration.remaining'
    gitlab-rails runner -e production 'puts Gitlab::Database::BackgroundMigration::BatchedMigration.queued.count'
  Docker install
    docker exec -t gitlab gitlab-rails runner -e production 'puts Gitlab::BackgroundMigration.remaining'
    docker exec -t gitlab gitlab-rails runner -e production 'puts Gitlab::Database::BackgroundMigration::BatchedMigration.queued.count'
Both of these commands regualy take around 2 minutes to give back an awnser and should give back 0. 

Than it creates a complete backup via rails. 
The commands to do so are :
  Omnibus install
    gitlab-backup create
  Docker install
    docker exec -t gitlab gitlab-backup create
This will take around 5 hours. 
When this is finished, we can upgrade the application. 

The commands to do so are :
  Omnibus install
    dnf install -y gitlab
  Docker install
    cd /opt/gitlab && docker-compose pull && docker-compose up -d
'/opt/gitlab' only an example value, this should be again set via enviroment variables. 
when not, try to run the golang eqvivalent of 
`docker inspect gitlab   | grep 'com.docker.compose.project.working_dir'`
        "com.docker.compose.project.working_dir": "/opt/gitlab",
Extract out of the above command the /opt/gitlab. 
When this fails, error out with re request to write a pull request for installations via plain docker commands.

Finally after upgrade the post upgrade check returns basically the same as the pre upgrade check.