Gitlab Autoscaling Infrastructure on Hcloud with internal caching

README.md

Order and provision a Hetzner Cloud based Gitlab-Runner Docker-machine autoscaling infrastructure

See my blog for more information.

• Adjust settings in vars.auto.tfvars.
• Run with terraform init && terraform apply

Content:

• order volume + smallest HCloud VM (~3 EUR/month)
• formats and mounts the volume on first boot
• volume is used for Docker registry proxy and S3-Proxy for shared caching.
• installs docker, docker-machine, and hcloud plugin for docker-machine.
• afterwards, registers Runner at your Gitlab instance with Off-Peak feature enabled (meaning in the night time it scales down.) See cloudinit.yml:112.
• on destroy, unregister runner and delete all running docker machine minions.

cloudinit.yml

#cloud-config
groups:
- docker
users:
- name: gitlab-runner
  groups: docker
apt:
  sources:
    docker.list:
      source: 'deb [arch=amd64] https://download.docker.com/linux/ubuntu $RELEASE stable'
      keyid: 0EBFCD88 # GPG key ID published on a key server
    gitlab.list:
      source: 'deb https://packages.gitlab.com/runner/gitlab-runner/ubuntu/ $RELEASE main'
      keyid: "0x14219A96E15E78F4"
      keyserver: keyserver.ubuntu.com
    gitlab2.list:
      keyid: "0x3CFCF9BAF27EAB47"
      keyserver: keyserver.ubuntu.com

package_upgrade: true
package_update: true
packages:
- debian-archive-keyring
- apt-transport-https
- ca-certificates
- software-properties-common
- htop
- docker-ce
- golang-go
- gitlab-runner
- fail2ban
- vim
- curl
- tar
- jq

write_files:
  - owner: 'root:root'
    path: /root/s3_cache.sh
    content: |
      volume=$(ls /dev/disk/by-id/scsi-0HC_Volume*)
      yes | sudo mkfs.ext4 -F $volume
      mkdir /export
      mount -o discard,defaults $volume /export
      echo "$volume /export ext4 discard,nofail,defaults 0 0" >> /etc/fstab
      systemctl daemon-reload

      mkdir /export/runner
      docker run -it -d --restart always -p 9005:9000 \
        -v /.minio:/root/.minio -v /export:/export \
        --name minio \
        minio/minio:latest server /export
      sleep 40
      echo "export MY_IP=$(ip addr show eth0 | grep -Po 'inet \K[\d.]+')" >> ~/.profile
      echo "export CACHE_S3_SERVER_ADDRESS=$(ip addr show eth0 | grep -Po 'inet \K[\d.]+'):9005" >> ~/.profile
      echo "export CACHE_TYPE=s3" >> ~/.profile
      echo "export CACHE_SHARED=true" >> ~/.profile
      echo "export CACHE_S3_BUCKET_NAME=runner" >> ~/.profile
      echo "export CACHE_S3_INSECURE=true" >> ~/.profile
      echo "export CACHE_S3_ACCESS_KEY=$(cat /export/.minio.sys/config/config.json | jq -r '.credential.accessKey')" >> ~/.profile
      echo "export CACHE_S3_SECRET_KEY=$(cat /export/.minio.sys/config/config.json | jq -r '.credential.secretKey')" >> ~/.profile

 - owner: 'root:root'
    path: /root/docker_proxy.sh
    content: |
      docker run -d -p 6000:5000 \
        -e REGISTRY_PROXY_REMOTEURL=https://registry-1.docker.io \
        -v /export/registry:/var/lib/registry \
        --restart always \
        --name registry registry:2

  - owner: 'root:root'
    path: /root/download_driver.sh
    content: |
      echo "Downloading docker-machine"
      url=$(curl --silent https://api.github.com/repos/docker/machine/releases | jq -r '. | first | .assets[] | select(.name|contains("Linux-x86_64")).browser_download_url')
      wget -q $url -O /usr/bin/docker-machine
      chmod +x /usr/bin/docker-machine

      echo "Downloading docker-machine-driver-hetzner"
      mkdir /build
      cd /build
      url=$(curl --silent https://api.github.com/repos/JonasProgrammer/docker-machine-driver-hetzner/releases | jq -r '. | first | .assets[] | select(.name|contains("linux_amd64")).browser_download_url')
      curl -sL $url | tar xvz
      chmod +x docker-machine-driver-hetzner
      mv docker-machine-driver-hetzner /usr/local/bin/
  - owner: 'root:root'
    path: /root/register.sh
    content: |
      source ~/.profile
      gitlab-runner register \
        --executor docker+machine \
        -u ${gitlab_url} \
        --run-untagged \
        --tag-list ${tag_list} \
        --locked=false \
        --non-interactive \
        -r ${registration_token} \
        --docker-privileged=true \
        --docker-pull-policy=if-not-present --docker-shm-size=268435456 \
        --docker-volumes='/cache' \
        --docker-image="${default_image}" \
        --machine-idle-nodes 1 \
        --machine-idle-time 1800 \
        --machine-max-builds 100 \
        --machine-machine-driver "hetzner" \
        --machine-machine-name "runner-%s" \
        --machine-machine-options "hetzner-api-token=${hcloud_token}" \
        --machine-machine-options "hetzner-image=${autoscaling_os}" \
        --machine-machine-options "hetzner-server-type=${autoscaling_server}" \
        --machine-machine-options "engine-registry-mirror=http://$MY_IP:6000" \
        --machine-off-peak-periods '* * 0-8,18-23 * * mon-fri *' \
        --machine-off-peak-periods '* * * * * sat,sun *' \
        --machine-off-peak-idle-count '0' \
        --machine-off-peak-timezone "Europe/Berlin" \
        --docker-allowed-images '*' \
        --docker-allowed-images '*/*' \
        --docker-allowed-images '*/*/*' \
        --docker-allowed-images '${allowed_images_extra}' \
        --docker-allowed-services 'redis:*' \
        --docker-allowed-services 'postgres:*' \
        --docker-allowed-services 'mysql:*' \
        --docker-allowed-services '${allowed_images_extra}'

      sed -i 's/concurrent = [0-9]\+/concurrent = ${autoscaling_concurrency}/' /etc/gitlab-runner/config.toml

runcmd:
  - [/bin/bash, /root/s3_cache.sh]
  - [/bin/bash, /root/docker_proxy.sh]
  - [/bin/bash, /root/download_driver.sh]
  - [/bin/bash, /root/register.sh]

power_state:
  delay: "now"
  mode: reboot
  message: First reboot
  condition: True
  
swap:
  filename: /var/swap.img
  size: "auto" # or size in bytes
  maxsize: 4294967296

# vim: ft=yaml

gitlab_autoscaling.tf

variable "autoscaling_server" {
  default = "cx31"
}
variable "autoscaling_os" {
  default = "ubuntu-18.04"
}
variable "autoscaling_concurrency" {
  default = 3
}

variable "hcloud_token" {}

provider "hcloud" {
  token = "${var.hcloud_token}"
}

resource "hcloud_ssh_key" "default" {
  name = "Terraform Key"
  public_key = "${file("keys/id_rsa.pub")}"
}

data "template_file" "broker_cloudinit" {
  template = "${file("templates/cloudinit.yml")}"
  vars = {
    gitlab_url = "${var.gitlab_instance_url}"
    registration_token = "${var.registration_token}"
    allowed_images_extra = "${var.allowed_images_extra}"
    default_image = "${var.default_image}"
    hcloud_token = "${var.hcloud_token}"
    tag_list = "ruby"
    autoscaling_os = "${var.autoscaling_os}"
    autoscaling_server = "${var.autoscaling_server}"
    autoscaling_concurrency = "${var.autoscaling_concurrency}"
  }
}
resource "hcloud_server" "autoscaling_boss" {
  count = 1
  name = "hcbroker-${count.index}"
  image = "ubuntu-18.04"
  server_type = "cx11"
  location = "nbg1"
  ssh_keys = ["${hcloud_ssh_key.default.id}"]
  user_data = "${data.template_file.broker_cloudinit.rendered}"

  provisioner "remote-exec" {
    inline = [
      "docker-machine kill `docker-machine ls -f '{{.Name}}'` || true",
      "yes | docker-machine rm `docker-machine ls -f '{{.Name}}'` || true",
      "gitlab-runner unregister --all-runners || true"
    ]
    when = "destroy"
    connection {
      type     = "ssh"
      user     = "root"
      private_key = "${file("keys/id_rsa")}"
    }
  }
}

resource "hcloud_volume" "cache_volume" {
  name = "gitlab-runner-cache"
  size = 50
  server_id = "${hcloud_server.autoscaling_boss.id}"
  format = "ext4"
  automount = true
}

output "broker_ssh_ip" {
  value = "ssh -F ssh.config root@${join(" ", hcloud_server.autoscaling_boss.*.ipv4_address)}"
}

vars.auto.tfvars

hcloud_token = "YOUR CLOUD API TOKEN"
autoscaling_server = "cx31"
autoscaling_os = "ubuntu-18.04"
autoscaling_concurrency = 3
gitlab_instance_url = "https://git.YOURCOMPANY.com"
allowed_images_extra = "registry.YOURCOMPANY.com/developers/docker-images/*"
default_image = "registry.YOURCOMPANY.com/developers/docker-images/base:latest"
registration_token = "YOUR GITLAB REGISTRATION TOKEN"