Terraform for provisioning a private container builder, see https://incident.io/blog/container-builder

README.md

Container builder

This gist is associated with the blog post "Deploying to production in <5m with our hosted container builder".

It contains the terraform code used to provision the builder instance. While it uses a private module instance_group, the code should be a useful starting point to figure out how to run this yourself.

We assume the instance is placed in a fresh GCP project.

container-builder-cloud-config.tpl

#cloud-config
# vim: syntax=yaml

# Provision a builder user which will permit access via the provided ssh key
users:
  - name: builder
    groups:
      - docker
    ssh_authorized_keys:
      - ${ssh_public_key}

# Create a docker group with root- trust the user provisioning to add builder
groups:
  - docker: [root]

runcmd:
  - mkfs.ext4 /dev/disk/by-id/google-docker || /bin/true
  - mount -a
  - apt-get update && apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release
  - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
  - echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
  - apt-get update && apt-get install -y docker-ce docker-ce-cli containerd.io
  - systemctl start docker

write_files:
  - path: "/etc/docker/daemon.json"
    permissions: "0644"
    owner: "root"
    content: |
      {
        "features": {
          "buildkit": true
        }
      }

main.tf

################################################################################
# CI/CD
################################################################################

terraform {
  required_providers {
    google = {
      source  = "hashicorp/google"
      version = "3.84.0"
    }
  }
}

provider "google" {
  region  = var.region
  project = var.project
}

################################################################################
# Variables
################################################################################

variable "project" {
  description = "Google Cloud Platform project name"
}
variable "environment" {
  description = "Unique environment name, lowercase, hyphenated and short"
}

variable "region" {
  description = "Default project region"
  default     = "europe-west2"
}

################################################################################
# Google Project
################################################################################

data "google_project" "project" {}

resource "google_project_service" "services" {
  for_each = toset([
    # Core resources
    "iam",
    # Compute resources
    "compute",
  ])
  service = "${each.key}.googleapis.com"
}

################################################################################
# Container builder
################################################################################

############################################################
# SSH key management
############################################################

# Used by CircleCI to access the builder user on the builder machine
resource "tls_private_key" "container_builder_circleci" {
  algorithm   = "ECDSA"
  ecdsa_curve = "P384"
}

# Sync the ssh key into Google Secret manager, so humans can access the key if
# they need to administrate CircleCI
resource "google_secret_manager_secret" "container_builder_circleci" {
  secret_id = "container-builder-circleci"
  replication {
    automatic = true
  }
}

resource "google_secret_manager_secret_version" "container_builder_circleci" {
  secret      = google_secret_manager_secret.container_builder_circleci.id
  secret_data = tls_private_key.container_builder_circleci.private_key_pem
}

# Provide developers access.
resource "google_secret_manager_secret_iam_binding" "container_builder_circleci" {
  secret_id = google_secret_manager_secret.container_builder_circleci.secret_id
  role      = "roles/secretmanager.secretAccessor"
  members = [
    "group:[email protected]",
  ]
}

############################################################
# Machine provisioning
############################################################

# Static IP address that is used by CircleCI to connect to the builder
resource "google_compute_address" "container_builder" {
  name         = "container-builder"
  address_type = "EXTERNAL"
}

module "container_builder" {
  source = "../../modules/instance_group"

  project = var.project
  region  = var.region

  prefix_name  = "container-builder"
  source_image = "ubuntu-2004-focal-v20210908"

  cloud_config = templatefile("${path.module}/container-builder-cloud-config.tpl", {
    ssh_public_key = tls_private_key.container_builder_circleci.public_key_openssh,
  })

  external_static_ips = [google_compute_address.container_builder.address]

  machine_type = "c2-standard-8" # Fast CPU, 8 vCPUs, 32 GB memory

  tags = ["container-builder"]

  mounts = [
    {
      name = "docker"
      path = "/var/lib/docker"
      size = 500 # GB
    },
  ]
}

Really cool! How are you dealing with cleaning up the docker disk as it fills up?