Auto upgrade Gitlab instance on Docker

README.md

Gitlab upgrade script

This script is a Node.js script for upgrading a Gitlab instance run on Docker. (tested with Node.js v14, but should run OK with some lower version - if someone can advise in the comments)

If you followed the Gitlab documentation to launch a Gitlab instance (https://docs.gitlab.com/ce/install/docker.html) by using the following command:

sudo docker run --detach \
--hostname gitlab.example.com \
--publish 443:443 --publish 80:80 --publish 22:22 \
--name gitlab \
--restart always \
--volume $GITLAB_HOME/config:/etc/gitlab \
--volume $GITLAB_HOME/logs:/var/log/gitlab \
--volume $GITLAB_HOME/data:/var/opt/gitlab \
gitlab/gitlab-ce:latest

You end up with a long process to update Gitlab. This script aims to automate the process by :

1. Checking the best version to install :
   1. Install patch version in priority
   2. Then, install minor version
   3. Then, install major version if allowed by the configuration (see bellow for more details)
2. Making a backup
3. Pulling the Docker image corresponding to the version to install
4. Restarting the container

This script can be run manually or can be setup via a cron task.

Installation

You will need to copy the 2 files gitlab-start and gitlab-upgrade on the server where your Gitlab container is running.

gitlab-start

This script is used to allow custom configuration of the Docker container. It must contain the command to run in order to start your Gitlab container. It must use the 2 environment variables that will be provided by the upgrade script : GITLAB_UPGRADE_CONTAINER_NAME and GITLAB_UPGRADE_VERSION_TAG.

The flags to use on the docker run command are up to you. The given script is an example that you must adapt to your case.

gitlab-upgrade

This script is used to upgrade your Gitlab instance. The only lines you need to change are the first 3 const. They are:

• CONTAINER_NAME: The name of the Gitlab container (default: "gitlab")
• ALLOW_MAJOR_VERSION_UPDATE: Are major upgrade allowed or not (default: false)
• START_COMMAND: The absolute path to the gitlab-start script (default: `${ __dirname }/gitlab-start`)

This config allows you to customize your setup. You may put the gitlab-start and the gitlab-upgrade script in different folders. The gitlab-upgrade must be registered in your PATH variable in order to be run from anywhere.

Version selection in details

The script only upgrade a single version at a time. So if you run it manually, you may need to run it several times to get a full upgrade.

The version selection is following the Gitlab recommendation, which is:

1. Install the highest path version corresponding to the current minor version
2. Otherwise, install the next minor version with the highest patch version
3. Otherwise, if allowed by configuration (ALLOW_MAJOR_VERSION_UPDATE), install the next major version with the first minor version (0) and the highest patch version

Example

I'm currently running with the version 13.10.2 and available versions are the following:

13.10	13.11	13.12	14.0	14.1
13.10.0	13.11.0	13.12.0	14.0.0	14.1.0
13.10.1	13.11.1	13.12.1	14.0.1
13.10.2	13.11.2	13.12.2	14.0.2
13.10.3	13.11.3	13.12.3	14.0.3
13.10.4	13.11.4	13.12.4	14.0.4
13.10.5	13.11.5	13.12.5	14.0.5
	13.11.6	13.12.6	14.0.6
	13.11.7	13.12.7
		13.12.8

By running the script several times (with ALLOW_MAJOR_VERSION_UPDATE=false), I'll have the following upgrades:

1. First run: upgrade from 13.10.2 to 13.10.5 (max patch version)
2. Second run: upgrade from 13.10.5 to 13.11.7 (next minor version)
3. Third run: upgrade from 13.11.7 to 13.12.8 (next minor version)
4. Fourth run: nothing to upgrade

Now, if I set ALLOW_MAJOR_VERSION_UPDATE=true and run again:

1. Fifth run: upgrade from 13.12.8 to 14.0.6 (next major version)
2. Sixth run: upgrade from 14.0.6 to 14.1.0 (next minor version)
3. Seventh run: nothing to upgrade

gitlab-start

#!/bin/bash

if [[ -z "$GITLAB_UPGRADE_CONTAINER_NAME" ]]; then
    echo "Env var GITLAB_UPGRADE_CONTAINER_NAME must be defined." 1>&2
    exit 1
fi
if [[ -z "$GITLAB_UPGRADE_VERSION_TAG" ]]; then
    echo "Env var GITLAB_UPGRADE_VERSION_TAG must be defined." 1>&2
    exit 1
fi

docker run --detach \
    --hostname gitlab.example.com \
    --publish 443:443 --publish 80:80 --publish 22:22 \
    --name $GITLAB_UPGRADE_CONTAINER_NAME \
    --restart always \
    --volume $GITLAB_HOME/config:/etc/gitlab \
    --volume $GITLAB_HOME/logs:/var/log/gitlab \
    --volume $GITLAB_HOME/data:/var/opt/gitlab \
    gitlab/gitlab-ce:$GITLAB_UPGRADE_VERSION_TAG

gitlab-upgrade

#!/usr/bin/env node

const DOCKER_COMMAND = "docker"
const CONTAINER_NAME = "gitlab"
const ALLOW_MAJOR_VERSION_UPDATE = false
const START_COMMAND = `${ __dirname }/gitlab-start`



const { spawn } = require("child_process")
const https = require("https")

async function main() {
    let current_version = await get_current_gitlab_version()
    console.log(`Current version installed: ${ current_version.tag }`)

    console.log("=== Fetching next version to install ===")
    let available_versions = await get_available_gitlab_versions(current_version.tag)
    let next_version = resolve_best_next_version(current_version, available_versions)
    if (next_version === null) {
        console.log(`Installed version (${ current_version.tag }) is the most recent one. No update required.`)
        console.log("=== Terminated with success ===")
        return
    }
    console.log(`=== Preparing installation of version ${ next_version.tag } ===`)

    console.log("=== Backup current instance ===")
    await execute_backup()

    console.log("=== Pull new container ===")
    await execute_pull_container(next_version)

    console.log("=== Restart with new container ===")
    await execute_current_container_stop()
    await execute_remove_current_container()
    await execute_start_container(next_version)
    let stop_logs = display_logs()
    await wait_for_container_to_start()
    stop_logs()

    console.log("=== Terminated with success ===")
}

/// Extract the current version of Gitlab from the running container by using `docker inspect` command.
async function get_current_gitlab_version() {
    // Run `docker inspect` command and capture output.
    let output = await new Promise((resolve, reject) => {
        // `docker inspect $CONTAINER_NAME`
        let command_process = spawn(DOCKER_COMMAND, ["inspect", CONTAINER_NAME])
        let output = ""
        command_process.stdout.on("data", data => {
            output += data.toString()
        })
        command_process.on("close", code => {
            if (code !== 0) {
                reject(new Error(`Could not get current installed version of Gitlab. Make sure the "${ CONTAINER_NAME }" container is running.`))
            }
            else {
                resolve(output)
            }
        })
    })

    // Parse previous command output and read Docker image tag.
    let content = JSON.parse(output)
    if (!content[0]?.Config?.Image) {
        throw new Error(`Could not extract image tag from container "${ CONTAINER_NAME }". Make sure "${ CONTAINER_NAME }" is the name of a container, not an image.`)
    }
    let version_match = content[0].Config.Image.match(/^gitlab\/gitlab-ce:(([\d.]+)-ce\.\d+)$/)
    if (version_match === null) {
        throw new Error(`Could not extract Gitlab version from tag "${ content[0].Config.Image }".`)
    }

    // Parse the version numbers.
    let version_parts = version_match[2].split(".")
    if (version_parts.length !== 3) {
        throw new Error(`Invalid format from version extracted: "${ version_match[2] }".`)
    }

    return {
        tag: version_match[1],
        major: parseInt(version_parts[0], 10),
        minor: parseInt(version_parts[1], 10),
        patch: parseInt(version_parts[2], 10),
    }
}

/// Fetch available tags of Gitlab-ce from docker hub API until it reaches the given version.
async function get_available_gitlab_versions(until_version) {
    let tags = []
    let url = "https://hub.docker.com/v2/repositories/gitlab/gitlab-ce/tags"

    while (url !== null && !tags.includes(until_version)) {
        // Query docker hub API.
        let response = await new Promise((resolve, reject) => {
            https.get(url, (response) => {
                let output = ""
                response.on("data", data => output += data)
                response.on("end", () => resolve(JSON.parse(output)))
                response.on("error", reject)
            })
        })

        // Save next page url.
        url = response.next

        // Extract tag list.
        response.results.forEach(tag => {
            if (tag.name.match(/^[\d.]+-ce\.\d+$/)) {
                tags.push(tag.name)
            }
        })
    }

    return tags.map(tag => {
        let match = tag.match(/^([\d.]+)-ce\.\d+$/)
        let parts = match[1].split(".")

        return {
            tag,
            major: parseInt(parts[0], 10),
            minor: parseInt(parts[1], 10),
            patch: parseInt(parts[2], 10),
        }
    })
}

/// Resolve the best next version from given current version and available versions with the following rules:
///   1. If a new patch version exists (same major, same minor, higher patch), it is selected.
///   2. Otherwise, if the next minor version is available (same major, minor + 1, any patch), it is selected.
///   3. Otherwise, if ALLOWED BY CONF and if the next major version is available (major + 1, minor = 0, any patch), it is selected.
///   4. Otherwise, null is returned: there are no available updates.
function resolve_best_next_version(current_version, available_versions) {
    // Get next patch version if available.
    let matching_versions = available_versions
        .filter(version => version.major === current_version.major && version.minor === current_version.minor)
    let best_patch_version = matching_versions[0]
    if (best_patch_version.tag !== current_version.tag) {
        return best_patch_version
    }

    // Get next minor version if available.
    let next_minor_version = current_version.minor + 1
    matching_versions = available_versions
        .filter(version => version.major === current_version.major && version.minor === next_minor_version)
    let best_minor_version = matching_versions[0]
    if (best_minor_version) {
        return best_minor_version
    }

    if (ALLOW_MAJOR_VERSION_UPDATE) {
        // Get next major version if available.
        let next_major_version = current_version.major + 1
        matching_versions = available_versions
            .filter(version => version.major === next_major_version && version.minor === 0)
        let best_major_version = matching_versions[0]
        if (best_major_version) {
            return best_major_version
        }
    }

    return null
}

/// Make a backup by executing `gitlab-rake gitlab:backup:create` command in the running container.
async function execute_backup() {
    return new Promise((resolve, reject) => {
        // `docker exec -t $CONTAINER_NAME gitlab-rake gitlab:backup:create`
        let command_process = spawn(DOCKER_COMMAND, ["exec", "-t", CONTAINER_NAME, "gitlab-rake", "gitlab:backup:create"])
        command_process.stdout.pipe(process.stdout)
        command_process.stderr.pipe(process.stderr)
        command_process.on("close", code => {
            if (code !== 0) {
                reject(new Error(`Could not make a backup from the current Gitlab container. Process terminated with status code ${ code }. Stopping the update process.`))
            }
            else {
                resolve()
            }
        })
    })
}

/// Pull the Gitlab container with the given version.
async function execute_pull_container(version) {
    return new Promise((resolve, reject) => {
        // `docker pull gitlab/gitlab-ce:$VERSION`
        let command_process = spawn(DOCKER_COMMAND, ["pull", `gitlab/gitlab-ce:${ version.tag }`])
        command_process.stdout.pipe(process.stdout)
        command_process.stderr.pipe(process.stderr)
        command_process.on("close", code => {
            if (code !== 0) {
                reject(new Error(`Could not pull the new Gitlab container. Process terminated with status code ${ code }. Stopping the update process.`))
            }
            else {
                resolve()
            }
        })
    })
}

/// Stop the running Gitlab container.
async function execute_current_container_stop() {
    return new Promise((resolve, reject) => {
        // `docker stop $CONTAINER_NAME`
        let command_process = spawn(DOCKER_COMMAND, ["stop", CONTAINER_NAME])
        command_process.stdout.pipe(process.stdout)
        command_process.stderr.pipe(process.stderr)
        command_process.on("close", code => {
            if (code !== 0) {
                reject(new Error(`Could not stop the running Gitlab container. Process terminated with status code ${ code }. Stopping the update process.`))
            }
            else {
                resolve()
            }
        })
    })
}

/// Remove the current Gitlab container.
async function execute_remove_current_container() {
    return new Promise((resolve, reject) => {
        // `docker container rm $CONTAINER_NAME`
        let command_process = spawn(DOCKER_COMMAND, ["container", "rm", CONTAINER_NAME])
        command_process.stdout.pipe(process.stdout)
        command_process.stderr.pipe(process.stderr)
        command_process.on("close", code => {
            if (code !== 0) {
                reject(new Error(`Could not remove the current Gitlab container. Process terminated with status code ${ code }. Stopping the update process.`))
            }
            else {
                resolve()
            }
        })
    })
}

/// Start the Gitlab container of the given version.
async function execute_start_container(version) {
    return new Promise((resolve, reject) => {
        let command_process = spawn(START_COMMAND, [], {
            env: {
                ...process.env,
                GITLAB_UPGRADE_CONTAINER_NAME: CONTAINER_NAME,
                GITLAB_UPGRADE_VERSION_TAG: version.tag,
            },
        })
        command_process.stdout.pipe(process.stdout)
        command_process.stderr.pipe(process.stderr)
        command_process.on("close", code => {
            if (code !== 0) {
                reject(new Error(`Could not start the Gitlab container. Process terminated with status code ${ code }. Stopping the update process.`))
            }
            else {
                resolve()
            }
        })
    })
}

/// Display the logs.
function display_logs() {
    // `docker logs -f $CONTAINER_NAME`
    let command_process = spawn(DOCKER_COMMAND, ["logs", "-f", CONTAINER_NAME])
    command_process.stdout.pipe(process.stdout)
    command_process.stderr.pipe(process.stderr)

    return function() {
        command_process.kill()
    }
}

/// Wait for the container to start.
async function wait_for_container_to_start() {
    return new Promise((resolve, reject) => {
        let interval = setInterval(() => {
            // `docker inspect --format={{.State.Health.Status}} $CONTAINER_NAME`
            let command_process = spawn(DOCKER_COMMAND, ["inspect", "--format={{.State.Health.Status}}", CONTAINER_NAME])
            let output = ""
            command_process.stdout.on("data", (data) => {
                output += data
            })
            command_process.stderr.pipe(process.stderr)
            command_process.on("close", code => {
                if (code !== 0) {
                    reject(new Error(`Could not inspect the starting Gitlab container. Process terminated with status code ${ code }. Stopping the update process.`))
                }
                else {
                    if (output.trim() === "healthy") {
                        clearInterval(interval)
                        resolve()
                    }
                }
            })
        }, 1000)
    })
}

main()
.catch(error => {
    console.error(error)
    process.exit(1)
})