GCE install for Buildkite Agent v3 (Beta)

setup.sh

#!/usr/bin/env bash

set -eo pipefail

DATE=$(date +%Y%m%d-%H%M%S)
GROUP_NAME="buildkite-agent"
TEMPLATE_NAME="${GROUP_NAME}-${DATE}"

# Builder Group
gcloud compute instance-templates create "${TEMPLATE_NAME}-builder" \
    --machine-type n1-standard-4 \
    --image-family debian-9 \
    --image-project debian-cloud \
    --boot-disk-size 100GB \
    --boot-disk-type pd-ssd \
    --service-account <replace_me> \
    --scopes cloud-platform \
    --region us-east1 \
    --metadata agent_role=docker-builder,agent_token=<replace_me>,github_credentials=<replace_me> \
    --metadata-from-file startup-script=startup \
    --preemptible

gcloud compute instance-groups managed create "${GROUP_NAME}-builder" \
  --template "${TEMPLATE_NAME}-builder" \
  --region us-east1 \
  --size 1

# Runner Group
gcloud compute instance-templates create "${TEMPLATE_NAME}-runner" \
    --machine-type n1-standard-16 \
    --image-family debian-9 \
    --image-project debian-cloud \
    --boot-disk-size 100GB \
    --boot-disk-type pd-ssd \
    --service-account <replace_me> \
    --scopes cloud-platform \
    --region us-east1 \
    --metadata agent_role=docker-runner,agent_token=<replace_me>,github_credentials=<replace_me> \
    --metadata-from-file startup-script=startup \
    --preemptible

gcloud compute instance-groups managed create "${GROUP_NAME}-runner" \
  --template "${TEMPLATE_NAME}-runner" \
  --zone us-east1-d \
  --size 2

startup

#!/bin/bash

set -eou pipefail

export DEBIAN_FRONTEND='noninteractive'
echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selections

DOCKER_RELEASE="stable"
DOCKER_VERSION=17.12.1-ce
DOCKER_COMPOSE_VERSION=1.19.0
DOCKER_CREDENTIALS_VERSION=1.4.3
BUILDKITE_AGENT_VERSION=3.0-beta.40-2155

BUILDKITE_AGENT_TOKEN=$(curl -sH "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/attributes/agent_token")
BUILDKITE_AGENT_ROLE=$(curl -sH "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/attributes/agent_role")
GITHUB_CREDENTIALS=$(curl -sH "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/attributes/github_credentials")

sudo apt-get update &&\
sudo apt-get install -qq -y --no-install-recommends \
     apt-transport-https \
     ca-certificates \
     curl \
     dirmngr \
     gnupg2 \
     software-properties-common

# Stackdriver Monitoring

curl -sSO https://dl.google.com/cloudagents/install-monitoring-agent.sh
sudo bash install-monitoring-agent.sh


# Buildkite install

sudo sh -c 'echo deb https://apt.buildkite.com/buildkite-agent unstable main > /etc/apt/sources.list.d/buildkite-agent.list'
sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 32A37959C2FA5C3C99EFBC32A79206696452D198

sudo apt-get update && sudo apt-get install -qq -y --no-install-recommends buildkite-agent=${BUILDKITE_AGENT_VERSION}

sudo sed -i "s/xxx/${BUILDKITE_AGENT_TOKEN}/g" /etc/buildkite-agent/buildkite-agent.cfg

echo tags="queue=${BUILDKITE_AGENT_ROLE}" | sudo tee -a /etc/buildkite-agent/buildkite-agent.cfg

sudo systemctl stop buildkite-agent && sudo systemctl disable buildkite-agent


# Docker CE install
#
sudo groupadd docker

wget -qO - "https://download.docker.com/linux/static/${DOCKER_RELEASE}/x86_64/docker-${DOCKER_VERSION}.tgz" | sudo tar xzf - --strip-components=1 -C /usr/bin
wget -qO - "https://raw.githubusercontent.com/moby/moby/master/contrib/init/systemd/docker.service" | sudo tee /lib/systemd/system/docker.service
wget -qO - "https://raw.githubusercontent.com/moby/moby/master/contrib/init/systemd/docker.socket" | sudo tee /lib/systemd/system/docker.socket


# Configure docker daemon
sudo mkdir /etc/docker
cat << EOF | sudo tee /etc/docker/daemon.json
{
  "storage-driver": "overlay2"
}
EOF

cat << EOF | sudo tee /etc/subuid
buildkite-agent:$(id -u buildkite-agent):1
buildkite-agent:100000:65536
EOF

cat << EOF | sudo tee /etc/subgid
buildkite-agent:$(getent group docker | cut -d: -f3):1
buildkite-agent:100000:65536
EOF

sudo systemctl daemon-reload
sudo systemctl enable docker
sudo systemctl restart docker

# Allow Buildkite-Agent to use Docker
sudo usermod -aG docker buildkite-agent


# GCR access

wget -qO - "https://github.com/GoogleCloudPlatform/docker-credential-gcr/releases/download/v${DOCKER_CREDENTIALS_VERSION}/docker-credential-gcr_linux_amd64-${DOCKER_CREDENTIALS_VERSION}.tar.gz" | sudo tar xzf - -C /usr/local/bin
sudo chmod 755 /usr/local/bin/docker-credential-gcr

# Allow Buildkite-Agent to access GCR
sudo -u buildkite-agent docker-credential-gcr configure-docker


# Docker Compose

sudo curl -sSL "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
sudo chmod 755 /usr/local/bin/docker-compose


# Git Credentials
sudo mkdir /etc/buildkite-agent/secrets/
echo "https://${GITHUB_CREDENTIALS}@github.com" | sudo tee /etc/buildkite-agent/secrets/git-credentials
sudo chown -R buildkite-agent: /etc/buildkite-agent/secrets/
git config --system credential.helper "store --file=/etc/buildkite-agent/secrets/git-credentials"

# Create a systemd template
sudo cp /lib/systemd/system/[email protected] /etc/systemd/system/[email protected]

# Run multiple Buildkite agents based on number of CPUs available
for i in $(seq 1 "$( grep -c ^processor /proc/cpuinfo )"); do
  sudo systemctl enable "buildkite-agent@$i" && sudo systemctl start "buildkite-agent@$i"
done


cat <<'EOF' | sudo tee /usr/local/bin/bk-check-disk-space.sh
#!/bin/bash
set -euo pipefail

DISK_MIN_AVAILABLE=${DISK_MIN_AVAILABLE:-1048576} # 1GB
DISK_MIN_INODES=${DISK_MIN_INODES:-10000}

disk_avail=$(df -k --output=avail "$PWD" | tail -n1)

echo "Disk space free: $(df -k -h --output=avail "$PWD" | tail -n1 | sed -e 's/^[[:space:]]//')"

if [[ ${disk_avail} -lt $DISK_MIN_AVAILABLE ]]; then
  echo "Not enough disk space free, cutoff is ${DISK_MIN_AVAILABLE}" >&2
  return 1
fi

inodes_avail=$(df -k --output=iavail "$PWD" | tail -n1)

echo "Inodes free: $(df -k -h --output=iavail "$PWD" | tail -n1 | sed -e 's/^[[:space:]]//')"

if [[ ${inodes_avail} -lt $DISK_MIN_INODES ]]; then
  echo "Not enough inodes free, cutoff is ${DISK_MIN_INODES inodes}" >&2
  return 1
fi

EOF

sudo chmod +x /usr/local/bin/bk-check-disk-space.sh


cat <<'EOF' | sudo tee /etc/cron.hourly/docker-low-disk-gc
#!/bin/bash
set -euo pipefail

DOCKER_PRUNE_UNTIL=${DOCKER_PRUNE_UNTIL:-1h}

## -----------------------------------------------------------------
## Check disk, we only want to prune images/containers if we have to

if ! /usr/local/bin/bk-check-disk-space.sh ; then
  echo "Cleaning up docker resources older than ${DOCKER_PRUNE_UNTIL}"
  docker image prune --all --force --filter "until=${DOCKER_PRUNE_UNTIL}"

  if ! /usr/local/bin/bk-check-disk-space.sh ; then
    echo "Disk health checks failed" >&2
    exit 1
  fi
fi

EOF

sudo chmod +x /etc/cron.hourly/docker-low-disk-gc

# Clean up Docker networks
cat <<'EOF' | sudo tee /etc/cron.hourly/docker-gc
#!/bin/bash

set -euo pipefail

DOCKER_PRUNE_UNTIL=${DOCKER_PRUNE_UNTIL:-1h}

## ------------------------------------------
## Prune stuff that doesn't affect cache hits

docker network prune --force --filter "until=${DOCKER_PRUNE_UNTIL}"
docker container prune --force --filter "until=${DOCKER_PRUNE_UNTIL}"

EOF

sudo chmod +x /etc/cron.hourly/docker-gc


# Fix permissions on files and directories
echo 'buildkite-agent ALL=NOPASSWD: /usr/local/bin/fix-buildkite-agent-builds-permissions' | sudo tee -a /etc/sudoers

cat <<'EOF' | sudo tee /usr/local/bin/fix-buildkite-agent-builds-permissions
#!/bin/bash

# To run the unit tests for this file, run the following command in the root of
# the project:
# $ docker-compose -f docker-compose.unit-tests.yml run unit-tests

# Files that are created by Docker containers end up with strange user and
# group ids, usually 0 (root). Docker namespacing will one day save us, but it
# can only map a single docker user id to a given user id (not any docker user
# id to a single system user id).
#
# Until we can map any old user id back to
# buildkite-agent automatically with Docker, then we just need to fix the
# permissions manually before each build runs so git clean can work as
# expected.

set -eu -o pipefail

# We need to scope the next bit to only the currently running agent dir and
# pipeline, but we also need to control security and make sure arbitrary folders
# can't be chmoded.
#
# We prepare the agent build directory basename in the environment hook and pass
# it as the first argument, org name as second argument, and the pipeline dir as
# the third.
#
# In here we need to check that they both don't contain slashes or contain a
# traversal component.

AGENT_DIR="$1"
# => "my-agent-1"

ORG_DIR="$2"
# => "my-org"

PIPELINE_DIR="$3"
# => "my-pipeline"

# Make sure it doesn't contain any slashes by substituting slashes with nothing
# and making sure it doesn't change
function exit_if_contains_slashes() {
  if [[ "${1//\//}" != "${1}" ]]; then
    exit 1
  fi
}

function exit_if_contains_traversal() {
  if [[ "${1}" == "." || "${1}" == ".." ]]; then
    exit 2
  fi
}

function exit_if_blank() {
  if [[ -z "${1}" ]]; then
    exit 3
  fi
}

# Check them for slashes
exit_if_contains_slashes "${AGENT_DIR}"
exit_if_contains_slashes "${ORG_DIR}"
exit_if_contains_slashes "${PIPELINE_DIR}"

# Check them for traversals
exit_if_contains_traversal "${AGENT_DIR}"
exit_if_contains_traversal "${ORG_DIR}"
exit_if_contains_traversal "${PIPELINE_DIR}"

# Check them for blank vaues
exit_if_blank "${AGENT_DIR}"
exit_if_blank "${ORG_DIR}"
exit_if_blank "${PIPELINE_DIR}"

# If we make it here, we're safe to go!

# We know the builds path:
BUILDS_PATH="/var/lib/buildkite-agent/builds"

# And now we can reconstruct the full agent builds path:
PIPELINE_PATH="${BUILDS_PATH}/${AGENT_DIR}/${ORG_DIR}/${PIPELINE_DIR}"
# => "/var/lib/buildkite-agent/builds/my-agent-1/my-org/my-pipeline"

if [[ -e "${PIPELINE_PATH}" ]]; then
  /bin/chown -R buildkite-agent:buildkite-agent "${PIPELINE_PATH}"
fi

EOF
sudo chmod +x /usr/local/bin/fix-buildkite-agent-builds-permissions

cat <<'EOF' | sudo tee /etc/buildkite-agent/hooks/environment
#!/bin/bash
set -eu -o pipefail

echo "Checking disk space"
if ! /usr/local/bin/bk-check-disk-space.sh ; then

  echo "Cleaning up docker resources older than ${DOCKER_PRUNE_UNTIL:-4h}"
  docker image prune --all --force --filter "until=${DOCKER_PRUNE_UNTIL:-4h}"

  echo "Checking disk space again"
  if ! /usr/local/bin/bk-check-disk-space.sh ; then
    echo "Disk health checks failed" >&2
    exit 1
  fi
fi

# So we can calculate the suffix as a substring:
AGENT_ORG_PIPELINE_DIR="${BUILDKITE_BUILD_CHECKOUT_PATH#${BUILDKITE_BUILD_PATH}/}"
# => "my-agent-1/my-org/my-pipeline"

# Then we can grab just the first path component, the agent name, by removing
# the longest suffix starting with a slash:
AGENT_DIR="${AGENT_ORG_PIPELINE_DIR%%/*}"
# => "my-agent-1"

# Then we can figure out the org/pipeline path component
ORG_PIPELINE_DIR="${AGENT_ORG_PIPELINE_DIR#${AGENT_DIR}/}"
# => "my-org/my-pipeline"

# Then we grab just the first path component, the org, by removing the longest
# suffix starting with a slash:
ORG_DIR="${ORG_PIPELINE_DIR%%/*}"
# => "my-org"

# Then we can figure out the pipeline path component using the org dir
PIPELINE_DIR="${ORG_PIPELINE_DIR#${ORG_DIR}/}"
# => "my-pipeline"

# Now we can pass this to the sudo script which will validate it before safely chmodding:
echo "Fixing permissions for '${AGENT_DIR}/${ORG_DIR}/${PIPELINE_DIR}'..."
sudo /usr/local/bin/fix-buildkite-agent-builds-permissions "${AGENT_DIR}" "${ORG_DIR}" "${PIPELINE_DIR}"
echo

EOF
sudo chmod +x /etc/buildkite-agent/hooks/environment
sudo chown buildkite-agent: /etc/buildkite-agent/hooks/environment