GKE Gitlab-runner with cloud-init

cloud-init-template.yaml

#cloud-config

package_update: true
package_upgrade: false # we will upgrade it with runcmd

fqdn: "${fqdn}"

apt:
  primary:
    - arches: [default]
      search_dns: True
  sources:
    gitlab.list:
      # https://packages.gitlab.com/install/repositories/runner/gitlab-runner/config_file.list?os=ubuntu&dist=bionic&source=script
      # https://packages.gitlab.com/gpg.key
      source: deb https://packages.gitlab.com/runner/gitlab-runner/ubuntu/ bionic main
      key: |
        -----BEGIN PGP PUBLIC KEY BLOCK-----

        mQINBF5dI2sBEACyGx5isuXqEV2zJGIx8rlJFCGw6A9g5Zk/9Hj50UpXNuOXlvQl
        7vq91m2CAh88Jad7OiMHIJJhX3ZJEOf/pUx/16QKumsaEyBk9CegxUG9jAQXsjL3
        WLyP0/l27UzNrOAFB+IUGjsoP+32gsSPiF5P485mirIJNojIAFzDQl3Uo4FbvqYU
        9AIRk5kV4nEYz1aKXAovIUsyqrztMtwlAG2xqdwVpGD2A4/w8I143qPGjjhEQmf4
        /EeS4CP9ztyLAx+01t2Acwa7Bygsb5KQPuT25UlevuxdDy/Rd5Zn/Lzwr2GQqjUs
        6GbM0t1HYjh57e4V+p0qMf6jxXfrDCbehgzFvGS0cx/d7hWHm5sXZIt3gxpjBQU2
        8MQWtrR8Y3nTBkCHwOKsXdsdD+YHxTq/yuvxl1Bcyshp29cGWv1es3wn2Z6i9tWe
        asGfVewJZiXFSEqSBGguEmLyCAZcWgXvHOV2kc66wG4d4TGIxmoo9GBqEtBftCVH
        MGDHt7zeg2hg6EIsx8/nj1duO5nBnbnik5iG8Xv46e/aw2p4DfTdfxHpjvyJudyN
        +UI5eSuuuXhyTZWedd5K1Q3+0CmACJ39t/NA6g7cZaw3boFKw3fTWIgOVTvC3y5v
        d7wsuyGUk9xNhHLcu6HjB4VPGzcTwQWMFf6+I4qGAUykU5mjTJchQeqmQwARAQAB
        tEJHaXRMYWIgQi5WLiAocGFja2FnZSByZXBvc2l0b3J5IHNpZ25pbmcga2V5KSA8
        cGFja2FnZXNAZ2l0bGFiLmNvbT6JAlQEEwEKAD4WIQT2QD9lRKOIY9qgtuA/AWGK
        UTEvPwUCXl0jawIbAwUJA8JnAAULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRA/
        AWGKUTEvP3/+EACEpR4JwFz7fAZxKoTzq1xkv7JiVC1jDnfZ6U6tumMDSIBLqlZX
        Jv/lixuXC/GCnyiduqqpO14RCkHrCECzNeu7+lt+eiBUpOKvDgkNid6FLMoulu1w
        hDhQWss6+40dIWwa5i8maIFg6WOwIiI24PW9T+ywrf2Gfv9mB1YP3ob+8Cx1EVb/
        sf5mu1SGHvq2PqNvPeyY3W5vU7rB0Ax5Kcn3e0Z+tUSC8fV7TCg9hm9o2Ou928K4
        hmvdFfR0t47cXt1wmZ/pjrWcezVqeIrMJyWtje4hgcO3TSXsfvedEdYn8Q/BgVRw
        9KL4DkR1HSemSsPB4YyOwLscjV6p5OCPm0PhPPXUGIdImcQH7jYuEXNi5nnN5dX4
        197ooTB2UCk8r0QtnhcQUE2ph46mylcksbR0nKhGh5bYW3jfd0X+MP36reo+EFQ7
        Sw35f7P7QvZqnEE8rd5fX3GImKm38xJi+9bGb4IH8WuslUZUMapgQqqBfw1k5+mP
        BBqKWSdEsP1i7LBv9jVOaauMYQPLZcodx5prgjrB89V1hCKu+ZQl/hzoCwmeSruD
        LUqX/RFeleZO2VeKXh1a/VQ69ThqZ7gyXcrvHopPPGTr9IESoV9/qcZWplEccP9b
        FuY9t6HuSpcL7SlbsRVQ0NBQrsQeZR2J0YgvRc3JWgZAfcE5MK2jcoWKCLkCDQRe
        XSNrARAApHc0R4tfPntr5bhTuXU/iVLyxlAlzdEv1XsdDC8YBYehT72Jpvpphtq7
        sKVsuC59l8szojgO/gW//yKSuc3Gm5h58+HpIthjviGcvZXf/JcN7Pps0UGkLeQN
        2+IRZgbA6CAAPh2njE60v5iXgS91bxlSJi8GVHq1h28kbKQeqUYthu9yA2+8J4Fz
        ivYV2VImKLSxbQlc86tl6rMKKIIOph+N4WujJgd5HZ80n2qp1608X3+9CXvtBasX
        VCI2ZqCuWjffVCOQzsqRbJ6LQyMbgti/23F4Yqjqp+8eyiDNL6MyWJCBbtkW3Imi
        FHfR0sQIM6I7fk0hvt9ljx9SG6az/s3qWK5ceQ7XbJgCAVS4yVixfgIjWvNE5ggE
        QNOmeF9r76t0+0xsdMYJR6lxdaQI8AAYaoMXTkCXX2DrASOjjEP65Oq/d42xpSf9
        tG6XIq+xtRQyFWSMc+HfTlEHbfGReAEBlJBZhNoAwpuDckOC08vw7v2ybS5PYjJ4
        5Kzdwej0ga03Wg9hrAFd/lVa5eO4pzMLuexLplhpIbJjYwCUGS4cc/LQ2jq4fue5
        oxDpWPN+JrBH8oyqy91b10e70ohHppN8dQoCa79ySgMxDim92oHCkGnaVyULYDqJ
        zy0zqbi3tJu639c4pbcggxtAAr0I3ot8HPhKiNJRA6u8HTm//xEAEQEAAYkCPAQY
        AQoAJhYhBPZAP2VEo4hj2qC24D8BYYpRMS8/BQJeXSNrAhsMBQkDwmcAAAoJED8B
        YYpRMS8/vzQP/iO0poRR9ZYjonP5GGIARRnF+xpWCRTZVSHLcAfS0ujZ7ekXoeeS
        JNMJ/7T4Yk1EJ9MTFZ83Jj4UybKO3Rw+/iPmcPpqUQGaEReYLlx7SyxmsOBXf+Q9
        PtyUmGO47tL+eAPInYyxsWGib/EeOw4KQrfByAIPWu0aeNeXadzxBLIkqD863H5q
        nTDrXOw6SLprlGt2zlc+XQKDv3DZez6wTcp205xdaNs55Bfk9pmKUS/ey3ZP7GvC
        CDEGxuWulVSKL2DYtq0sEZD7pQYSy8gBTqXLQAyfmPDcxe9Lczhk3UYrUUomN1/w
        +VE09q75yNqkaHdckVt+aYAHMgQ0ilmwTg6+OlEK+ZQkUT94viB6YW7B0M4uzols
        9FSDxXea/uKn75jTSkA3GAXf7O5hqbkDDctJbtO2pPdLDxbXN95iZ9xpgRE3exGl
        ucjgV5XGpLO4XXf0GTzug/TJAtNljJ/44+6meO0WwOwLMMhAJVxcp1fpbtgRmrcJ
        8bAsCkV5EO8SeQZDu2C8I9tMGlJ1VLTAfv6Lv2Z89B1AOOweGz4I48i9lux+HdXd
        HewnA37zx0XNjNQmqiG85UWUusnDxF0Je2jEhGIpHK/KdyI1BfNzX3d5HVoM1VE3
        THtRZHnetoMek8L5x/ciYQNIt40rQ6MHtPEo1ZC4346DP6eJmeX1DGGI
        =91uZ
        -----END PGP PUBLIC KEY BLOCK-----

write_files:
  - path: /etc/apt/preferences.d/pin-gitlab-runner.pref
    owner: root:root
    permissions: '0644'
    content: |
      Explanation: Prefer GitLab provided packages over the Debian native ones
      Package: gitlab-runner
      Pin: origin packages.gitlab.com
      Pin-Priority: 1001
  - path: /etc/gitlab-runner/gcs-creds.json
    owner: root:root
    permissions: '0640'
    encoding: b64
    content: "${cache-key}"
  - path: /etc/gitlab-runner/config.toml
    owner: root:root
    permissions: '0640'
    content: |
      concurrent = ${concurrent}
      check_interval = 0
      log_format = "text"
  - path: /etc/gitlab-runner/gcp-credentials.json
    owner: root:root
    permissions: '0640'
    encoding: b64
    content: "${machine-key}"
  - path: /etc/systemd/system/gitlab-runner.service.d/override.conf
    owner: root:root
    permissions: '0644'
    content: |
      [Service]
      Environment=GOOGLE_APPLICATION_CREDENTIALS=/etc/gitlab-runner/gcp-credentials.json

runcmd:
  - export DEBIAN_FRONTEND=noninteractive
  - apt-get remove -q -y --purge snapd lxd lxcfs liblxc-common liblxc1 unattended-upgrades tmux landscape-common language-selector-common snapd
  - apt-get autoremove -q -y
  - apt-get upgrade -q -y
  - apt-get install --no-install-recommends -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" -y docker.io gitlab-runner git tzdata less
  - ln -fs /usr/share/zoneinfo/Europe/Helsinki /etc/localtime
  - curl -L https://gitlab-docker-machine-downloads.s3.amazonaws.com/v0.16.2-gitlab.4/docker-machine > /usr/bin/docker-machine && chmod +x /usr/bin/docker-machine
  - >
    gitlab-runner register
    --non-interactive
    --url=https://gitlab.com/
    --registration-token=${registration-token}
    --name=${fqdn}
    --executor=docker+machine
    --docker-image=debian:buster-slim
    --docker-pull-policy=always
    --docker-privileged
    --machine-idle-nodes=1
    --machine-idle-time=600
    --machine-machine-driver=google
    --machine-machine-name="gitlab-autoscale-%s"
    --machine-machine-options="google-project=${machine-project}"
    --machine-machine-options="google-machine-type=${machine-type}"
    --machine-machine-options="google-zone=${machine-zone}"
    --machine-machine-options="google-machine-image=ubuntu-os-cloud/global/images/family/ubuntu-minimal-1804-lts"
    --machine-machine-options="google-tags=gitlab-runner-slave"
    --machine-machine-options="google-use-internal-ip=${use-internal-ip}"
    --machine-machine-options="google-preemptible=true"
    --machine-machine-options="google-min-cpu-platform=Intel Cascade Lake"
    --machine-machine-options="google-disk-size=30"
    --machine-machine-options="google-disk-type=pd-ssd"
    --machine-off-peak-periods="* * 0-9,16-23 * * mon-fri *"
    --machine-off-peak-periods="* * * * * sat,sun *"
    --machine-off-peak-timezone=Europe/Helsinki
    --machine-off-peak-idle-count=0
    --machine-off-peak-idle-time=120
    --cache-type=gcs
    --cache-shared
    --cache-path=gitlab-ci/runner-cache
    --cache-gcs-credentials-file=/etc/gitlab-runner/gcs-creds.json
    --cache-gcs-bucket-name=${cache-bucket}
  - systemctl enable gitlab-runner
  - systemctl disable docker

power_state:
  mode: reboot
  condition: True

main.tf

provider "google" {
}

provider "random" {
}

provider "template" {
}

data "google_compute_zones" "zones" {
  project = var.machine-project
  region  = var.machine-region
}

data "google_compute_image" "image" {
  family  = "ubuntu-minimal-2004-lts"
  project = "ubuntu-os-cloud"
}

resource "random_string" "instance" {
  count   = var.instances-count
  length  = 12
  special = false
  upper   = false
}

# add this option when gitlab-runner itself will be running inside google cloud
# --machine-machine-options="google-use-internal-ip=true"
locals {
  name-prefix = "gitlab-runner-"
  instances-names = [
    for i in range(0, var.instances-count) : {
      name     = "${local.name-prefix}${random_string.instance[i].result}"
      dns_name = "${local.name-prefix}${random_string.instance[i].result}.${var.dns-zone.dns_name}"
      fqdn     = "${local.name-prefix}${random_string.instance[i].result}.${replace(var.dns-zone.dns_name, "/\\.$/", "")}"
    }
  ]
  instances = [
    for i in range(0, var.instances-count) : {
      name     = local.instances-names[i].name
      dns_name = local.instances-names[i].dns_name
      user-data = templatefile("${path.module}/cloud-init.template.yaml", {
        fqdn               = local.instances-names[i].fqdn
        concurrent         = var.concurrent
        registration-token = var.registration-token
        cache-bucket       = var.cache-bucket
        cache-key          = var.cache-key
        machine-key        = var.machine-key
        machine-project    = var.machine-project
        machine-type       = var.machine-type
        machine-zone       = data.google_compute_zones.zones.names[i]
        use-internal-ip    = true
      })
      zone = data.google_compute_zones.zones.names[i]
    }
  ]
}

resource "google_compute_instance" "instance" {
  count        = length(local.instances)
  name         = local.instances[count.index].name
  machine_type = var.runner-machine-type
  project      = var.runner-project
  zone         = local.instances[count.index].zone

  boot_disk {
    initialize_params {
      size  = "20"
      image = data.google_compute_image.image.name
    }
  }

  network_interface {
    network = "default"

    access_config {
      // TODO: remove external IP from gitlab-runner and docker-machine instances
    }
  }

  metadata = {
    user-data   = local.instances[count.index].user-data
    server-role = "gitlab-runner"
    env         = "infra"
  }

  service_account {
    # copy-pasted form manually created instance
    scopes = [
      "https://www.googleapis.com/auth/devstorage.read_only",
      "https://www.googleapis.com/auth/logging.write",
      "https://www.googleapis.com/auth/service.management.readonly",
      "https://www.googleapis.com/auth/servicecontrol",
      "https://www.googleapis.com/auth/trace.append",
      "https://www.googleapis.com/auth/monitoring.write",
    ]
  }

  lifecycle {
    ignore_changes = [
      boot_disk[0].initialize_params,
    ]
  }

}

variables.tf

variable "runner-machine-type" {
  default = "f1-micro"
}
variable "runner-project" {
}

variable "instances-count" {
}

variable "dns-zone" {
}

variable "cache-bucket" {
}

variable "cache-key" {
}

variable "registration-token" {
}

variable "concurrent" {
}

variable "machine-project" {
  description = "google project to run machines by docker+machine executor"
}

variable "machine-key" {
  description = "google auth credentials to use by docker+machine executor"
}

variable "machine-region" {
}

# n2-standard-2: 2 cores (2.8GHz - 3.9GHz) 8GiB memory $0.0235/hour aka $17.16/month
# n2d-standard-2: 2 cores (2.25GHz-2.7GHz-3.3 GHz) 8GiB memory $0.0204/h aka $14.89/m
# c2-standard-4: 4 cores (3.1GHz - 3.9GHz) 16GiB memory $0.0505/hour aka $36.865/month
variable "machine-type" {
  default = "n2-standard-2"
}