crunch-server: ephemeral Hetzner Cloud box for remote builds (crunch for Rust, rsync+ssh for cmake/meson/make)

crunch-server

#!/usr/bin/env bash
# Bring up / tear down an ephemeral Hetzner Cloud box for remote builds.
# Rust goes through https://github.com/liamaharon/crunch; anything else
# (cmake, meson, make) through `run`, which uses the same rsync layout.
set -euo pipefail

NAME="${CRUNCH_NAME:-crunch}"
TYPE="${CRUNCH_TYPE:-ccx33}"
LOCATION="${CRUNCH_LOCATION:-fsn1}"
IMAGE="${CRUNCH_IMAGE:-debian-13}"
PUBKEY="${CRUNCH_PUBKEY:-$HOME/.ssh/id_ed25519.pub}"
SSH_HOST_ALIAS="${CRUNCH_SSH_ALIAS:-crunch}"
# Any command printing the token on stdout; swap for keepassxc-cli, secret-tool, etc.
TOKEN_CMD="${CRUNCH_TOKEN_CMD:-pass show hetzner/api-token}"

# Rough EUR/hour, only used for the accrued-cost readout in `status`.
declare -A HOURLY=([ccx13]=0.0555 [ccx23]=0.1109 [ccx33]=0.2219 [ccx43]=0.4423 [ccx53]=0.8545 [ccx63]=1.3671)

SSH_CONFIG="$HOME/.ssh/config"
KNOWN_HOSTS="$HOME/.ssh/known_hosts.crunch"
STATE_DIR="${XDG_STATE_HOME:-$HOME/.local/state}/crunch-server"
STATE_FILE="$STATE_DIR/$NAME.env"
BEGIN_MARK="# >>> crunch-server ($SSH_HOST_ALIAS) >>>"
END_MARK="# <<< crunch-server ($SSH_HOST_ALIAS) <<<"

die() { echo "error: $*" >&2; exit 1; }
info() { echo "==> $*"; }

hc() {
  if command -v hcloud >/dev/null 2>&1; then
    hcloud "$@"
  else
    guix shell hetznercloud-cli -- hcloud "$@"
  fi
}

require_token() {
  [ -n "${HCLOUD_TOKEN:-}" ] && return 0
  local t
  # First line only: `pass show` prints the secret first, then any metadata.
  t="$(eval "$TOKEN_CMD" 2>/dev/null | head -n1 || true)"
  if [ -n "$t" ]; then
    export HCLOUD_TOKEN="$t"
    return 0
  fi
  die "no Hetzner token. Either:
  pass insert -f ${TOKEN_CMD##* }   # then re-run
  export HCLOUD_TOKEN=...           # one-off
  CRUNCH_TOKEN_CMD='...'            # different secret backend
Create the token at Console > Security > API tokens (Read & Write)."
}

# Hetzner indexes SSH keys by MD5 fingerprint.
key_fingerprint() { ssh-keygen -lf "$1" -E md5 | awk '{print $2}' | sed 's/^MD5://'; }

ensure_ssh_key() {
  [ -f "$PUBKEY" ] || die "public key not found: $PUBKEY"
  local fp name
  fp="$(key_fingerprint "$PUBKEY")"
  name="$(hc ssh-key list -o noheader -o columns=name,fingerprint \
          | awk -v fp="$fp" '$2 == fp {print $1; exit}')"
  if [ -z "$name" ]; then
    name="$(basename "$PUBKEY" .pub)-$(hostname -s)"
    info "uploading $PUBKEY to the project as '$name'"
    hc ssh-key create --name "$name" --public-key-from-file "$PUBKEY" >/dev/null
  fi
  echo "$name"
}

cloud_init() {
  cat <<'YAML'
#cloud-config
package_update: true
packages:
  - rsync
  - git
  - build-essential
  - pkg-config
  - libssl-dev
  - libpq-dev
  - clang
  - lld
  - cmake
  - ninja-build
  - meson
  - ccache
  - protobuf-compiler
  - qtbase5-dev
  - qtbase5-dev-tools
  - qt5-qmake
  - libqt5svg5-dev
  - qt6-base-dev
  - qt6-base-dev-tools
  - libgtest-dev
  - libgmock-dev
  - nlohmann-json3-dev
  - libcurl4-openssl-dev
  - libqrcodegencpp-dev
runcmd:
  - [ sh, -c, "curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable" ]
  # Debian installs the qrcodegen C++ header under qrcodegen/, Guix under qrcodegencpp/.
  - [ sh, -c, "ln -sfn /usr/include/qrcodegen /usr/include/qrcodegencpp" ]
  # crunch runs cargo over a non-interactive ssh, which never sources profile scripts.
  - [ sh, -c, "ln -sf /root/.cargo/bin/* /usr/local/bin/" ]
  - [ sh, -c, "touch /var/lib/crunch-ready" ]
YAML
}

write_ssh_config() {
  local ip="$1" identity="${PUBKEY%.pub}"
  mkdir -p "$HOME/.ssh"; chmod 700 "$HOME/.ssh"
  touch "$SSH_CONFIG"; chmod 600 "$SSH_CONFIG"
  remove_ssh_config
  # Prepend: a leading `Host *` block elsewhere in the file would otherwise shadow these.
  local tmp; tmp="$(mktemp)"
  {
    echo "$BEGIN_MARK"
    echo "Host $SSH_HOST_ALIAS"
    echo "    HostName $ip"
    echo "    User root"
    echo "    IdentityFile $identity"
    echo "    IdentitiesOnly yes"
    echo "    StrictHostKeyChecking accept-new"
    echo "    UserKnownHostsFile $KNOWN_HOSTS"
    echo "    ControlMaster auto"
    echo "    ControlPath ~/.ssh/cm-%r@%h:%p"
    echo "    ControlPersist 5m"
    echo "    ServerAliveInterval 30"
    echo "$END_MARK"
    echo
    cat "$SSH_CONFIG"
  } >"$tmp"
  mv "$tmp" "$SSH_CONFIG"; chmod 600 "$SSH_CONFIG"
}

remove_ssh_config() {
  [ -f "$SSH_CONFIG" ] || return 0
  local tmp; tmp="$(mktemp)"
  awk -v b="$BEGIN_MARK" -v e="$END_MARK" '
    $0 == b {skip=1; next} $0 == e {skip=0; next} !skip' "$SSH_CONFIG" >"$tmp"
  mv "$tmp" "$SSH_CONFIG"; chmod 600 "$SSH_CONFIG"
}

cmd_up() {
  require_token
  # Reusing a live box is free; a fresh one costs another rounded-up hour.
  local existing
  if existing="$(hc server describe "$NAME" -o "format={{.ServerType.Name}}" 2>/dev/null)"; then
    [ "$existing" = "$TYPE" ] || die "'$NAME' is already up as $existing, not $TYPE. Reuse it as-is, or '$0 down' first."
    write_ssh_config "$(hc server ip "$NAME")"
    info "'$NAME' already up - reusing"
    cmd_status
    return 0
  fi

  local keyname; keyname="$(ensure_ssh_key)"

  info "creating $NAME ($TYPE, $IMAGE, $LOCATION)"
  cloud_init | hc server create \
    --name "$NAME" --type "$TYPE" --image "$IMAGE" --location "$LOCATION" \
    --ssh-key "$keyname" --user-data-from-file - >/dev/null

  local ip; ip="$(hc server ip "$NAME")"
  info "ip $ip - waiting for sshd"
  # Fresh IP from a recycled pool may carry a stale host key.
  ssh-keygen -R "$ip" -f "$KNOWN_HOSTS" >/dev/null 2>&1 || true
  write_ssh_config "$ip"

  local i
  for i in $(seq 1 60); do
    ssh -o ConnectTimeout=5 -o BatchMode=yes "$SSH_HOST_ALIAS" true 2>/dev/null && break
    [ "$i" = 60 ] && die "sshd never came up on $ip"
    sleep 5
  done

  info "waiting for cloud-init (installing rust toolchain, a few minutes)"
  ssh "$SSH_HOST_ALIAS" 'cloud-init status --wait >/dev/null 2>&1 || true'
  ssh "$SSH_HOST_ALIAS" 'test -f /var/lib/crunch-ready' \
    || die "cloud-init did not finish cleanly; check: ssh $SSH_HOST_ALIAS cat /var/run/cloud-init/result.json"

  mkdir -p "$STATE_DIR"
  printf 'CREATED=%s\nTYPE=%s\nIP=%s\n' "$(date +%s)" "$TYPE" "$ip" >"$STATE_FILE"

  echo
  info "ready: $(ssh "$SSH_HOST_ALIAS" 'cargo --version'), $(ssh "$SSH_HOST_ALIAS" 'g++ --version | head -n1'), $(ssh "$SSH_HOST_ALIAS" 'cmake --version | head -n1'), $(ssh "$SSH_HOST_ALIAS" 'nproc') vCPU"
  ssh "$SSH_HOST_ALIAS" 'lscpu | grep -E "^Model name"' || true
  echo
  echo "  crunch build --release                              # from a cargo project"
  echo "  $0 run -- cmake -B build -G Ninja '&&' cmake --build build   # anything else"
  echo "  $0 down                                              # BILLING RUNS UNTIL THIS"
}

# Same layout crunch uses for remote_path = "unique": ~/crunch-builds/<name>-<hash>.
remote_build_dir() {
  echo "crunch-builds/$(basename "$PWD")-$(printf '%s' "$PWD" | sha1sum | cut -c1-8)"
}

cmd_run() {
  local exclude="${CRUNCH_EXCLUDE:-.git,build,target}"
  local copy_back="${CRUNCH_COPY_BACK:-}"
  local -a envs=()
  # shellcheck disable=SC2206
  [ -n "${CRUNCH_ENV:-}" ] && envs=(${CRUNCH_ENV})
  while [ $# -gt 0 ]; do
    case "$1" in
      -x|--exclude)   exclude="$2"; shift 2 ;;
      -b|--copy-back) copy_back="$2"; shift 2 ;;
      -e|--env)       envs+=("$2"); shift 2 ;;
      --)             shift; break ;;
      -*)             die "run: unknown option $1" ;;
      *)              break ;;
    esac
  done
  [ $# -gt 0 ] || die "usage: $0 run [-x a,b] [-b path,..] [-e VAR=val] -- <command...>
The command is joined and run by the remote shell, so quote operators: '&&', '|', ';'."
  ssh -o ConnectTimeout=5 -o BatchMode=yes "$SSH_HOST_ALIAS" true 2>/dev/null \
    || die "'$SSH_HOST_ALIAS' is not reachable. Run '$0 up' first."

  local dir; dir="$(remote_build_dir)"
  local -a ex=() parts=()
  IFS=',' read -ra parts <<<"$exclude"
  local p; for p in "${parts[@]}"; do [ -n "$p" ] && ex+=(--exclude "$p"); done

  local -a progress=()
  [ -t 1 ] && progress=(--info=progress2)

  info "syncing $PWD -> $SSH_HOST_ALIAS:~/$dir (excluding: $exclude)"
  # Excluded paths (the build dir) survive --delete, so remote builds stay incremental.
  LC_ALL=C.UTF-8 rsync -a --delete --compress "${progress[@]}" "${ex[@]}" \
    --rsync-path "mkdir -p ~/$dir && rsync" "$PWD/" "$SSH_HOST_ALIAS:$dir"

  local exports="export MAKEFLAGS=-j\$(nproc) CMAKE_BUILD_PARALLEL_LEVEL=\$(nproc)"
  exports="$exports CMAKE_C_COMPILER_LAUNCHER=ccache CMAKE_CXX_COMPILER_LAUNCHER=ccache"
  local kv; for kv in "${envs[@]}"; do exports="$exports $(printf '%q' "$kv")"; done

  info "running on $SSH_HOST_ALIAS: $*"
  local status=0 tty=()
  # A pty gives colours and Ctrl-C forwarding, but ssh warns when stdin isn't a terminal.
  [ -t 0 ] && tty=(-t)
  # Keep the exit code for after copy-back.
  LC_ALL=C.UTF-8 ssh "${tty[@]}" "$SSH_HOST_ALIAS" "$exports; cd ~/$dir && $*" || status=$?

  if [ -n "$copy_back" ]; then
    IFS=',' read -ra parts <<<"$copy_back"
    for p in "${parts[@]}"; do
      [ -n "$p" ] || continue
      info "copying back $p"
      # -R with the /./ anchor recreates the path relative to the project root.
      LC_ALL=C.UTF-8 rsync -aR --compress "${progress[@]}" "$SSH_HOST_ALIAS:$dir/./$p" "$PWD/" \
        || echo "warning: could not copy back $p" >&2
    done
  fi
  [ "$status" = 0 ] || info "remote command exited with $status"
  return "$status"
}

cmd_apt() {
  [ $# -gt 0 ] || die "usage: $0 apt <package>..."
  ssh -t "$SSH_HOST_ALIAS" "DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends $*"
}

cmd_down() {
  require_token
  if hc server describe "$NAME" >/dev/null 2>&1; then
    info "deleting $NAME"
    hc server delete "$NAME" >/dev/null
  else
    info "no server named $NAME"
  fi
  ssh -O exit "$SSH_HOST_ALIAS" 2>/dev/null || true
  remove_ssh_config
  rm -f "$KNOWN_HOSTS" "$STATE_FILE"

  # Auto-created primary IPs normally go with the server; unassigned ones bill regardless.
  local orphans
  orphans="$(hc primary-ip list -o noheader -o columns=name,assignee 2>/dev/null | awk '$2 == "-" {print $1}' || true)"
  [ -n "$orphans" ] && echo "note: unassigned primary IPs still billing: $orphans"
  return 0
}

cmd_status() {
  require_token
  local out
  if ! out="$(hc server describe "$NAME" \
       -o "format={{.Name}}  {{.ServerType.Name}}  {{.Status}}  {{.PublicNet.IPv4.IP}}" 2>&1)"; then
    # Don't let a rejected token masquerade as "no server".
    case "$out" in
      *nauthorized*|*oken*) die "API rejected the token: $out" ;;
      *) echo "$NAME: not running"; return 0 ;;
    esac
  fi
  echo "$out"
  [ -f "$STATE_FILE" ] || return 0
  # shellcheck disable=SC1090
  . "$STATE_FILE"
  local hours rate
  hours=$(( ( $(date +%s) - CREATED + 3599 ) / 3600 ))
  rate="${HOURLY[$TYPE]:-0}"
  awk -v h="$hours" -v r="$rate" 'BEGIN{printf "up %dh (billed), ~EUR %.2f so far\n", h, h*r}'
}

case "${1:-}" in
  up)     cmd_up ;;
  down)   cmd_down ;;
  status) cmd_status ;;
  run)    shift; cmd_run "$@" ;;
  apt)    shift; cmd_apt "$@" ;;
  ssh)    shift; exec ssh "$SSH_HOST_ALIAS" "$@" ;;
  *) cat <<EOF
usage: $(basename "$0") {up|down|status|run|apt|ssh}

  up      create the server, install the toolchains, write the '$SSH_HOST_ALIAS' ssh config block
  down    delete the server and remove the config block (stops billing)
  status  show the server and accrued cost
  run [-x a,b] [-b path,..] [-e VAR=val] -- <command...>
          rsync the current directory to the box, run the command there, copy paths back.
          Rust projects use \`crunch\` instead; this is for cmake, meson, make, scripts.
          -x  paths to exclude from the upload (default: .git,build,target)
          -b  paths to rsync back afterwards, relative to the project (default: none)
          -e  environment for the remote command, repeatable
  apt <package>...
          install extra Debian packages on the running box
  ssh [cmd]
          open a shell or run a command on the box

env overrides: CRUNCH_NAME CRUNCH_TYPE CRUNCH_LOCATION CRUNCH_IMAGE CRUNCH_PUBKEY
               CRUNCH_SSH_ALIAS CRUNCH_TOKEN_CMD CRUNCH_EXCLUDE CRUNCH_COPY_BACK CRUNCH_ENV
defaults: $NAME / $TYPE / $LOCATION / $IMAGE / $PUBKEY
token:    \$HCLOUD_TOKEN, else \`$TOKEN_CMD\`
EOF
     exit 1 ;;
esac