Hardening CI/CD Pipelines: Ephemeral Self-Hosted Runner Orchestration on AWS EC2 with GitHub Actions OIDC and HashiCorp Vault — code snippets

snippet-1.hcl

# Configure the OIDC/JWT Auth Backend for GitHub Actions in HashiCorp Vault
resource "vault_jwt_auth_backend" "github" {
  description        = "JWT Auth backend for GitHub Actions OIDC tokens"
  path               = "github-actions"
  oidc_discovery_url = "https://token.actions.githubusercontent.com"
  bound_issuer       = "https://token.actions.githubusercontent.com"
}

# Define a policy that allows reading dynamic database credentials
resource "vault_policy" "production_db_read" {
  name   = "production-db-read"
  policy = <<EOT
path "database/creds/app-prod-role" {
  capabilities = ["read"]
}
path "secret/data/production/*" {
  capabilities = ["read"]
}
EOT
}

snippet-2.go

package main

import (
	"context"
	"crypto/hmac"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"io"
	"net/http"
	"os"

	"github.com/aws/aws-sdk-go-v2/config"
	"github.com/aws/aws-sdk-go-v2/service/ec2"
	"github.com/aws/aws-sdk-go-v2/service/ec2/types"
	"github.com/aws/smithy-go/ptr"
)

type WebhookPayload struct {
	Action      string `json:"action"` // "queued", "completed"
	WorkflowJob struct {
		ID     int64    `json:"id"`
		Labels []string `json:"labels"`
	} `json:"workflow_job"`
}

func main() {
	http.HandleFunc("/webhook", handleWebhook)
	http.ListenAndServe(":8080", nil)
}

func handleWebhook(w http.ResponseWriter, r *http.Request) {
	secret := []byte(os.Getenv("GITHUB_WEBHOOK_SECRET"))
	body, err := io.ReadAll(r.Body)
	if err != nil {
		http.Error(w, "Read error", http.StatusInternalServerError)
		return
	}

	// Validate HMAC-SHA256 signature
	signature := r.Header.Get("X-Hub-Signature-256")
	if !validateSignature(body, signature, secret) {
		http.Error(w, "Unauthorized signature", http.StatusUnauthorized)
		return
	}

	var payload WebhookPayload
	if err := json.Unmarshal(body, &payload); err != nil {
		http.Error(w, "Bad request", http.StatusBadRequest)
		return
	}

	// Only provision on "queued" events for our specific runner label
	if payload.Action == "queued" && contains(payload.WorkflowJob.Labels, "aws-ephemeral") {
		go launchRunnerInstance(r.Context(), payload.WorkflowJob.ID)
	}
	w.WriteHeader(http.StatusAccepted)
}

func validateSignature(payload []byte, signatureHeader string, secret []byte) bool {
	if len(signatureHeader) < 7 || signatureHeader[:7] != "sha256=" {
		return false
	}
	gotSig, err := hex.DecodeString(signatureHeader[7:])
	if err != nil {
		return false
	}
	mac := hmac.New(sha256.New, secret)
	mac.Write(payload)
	expectedSig := mac.Sum(nil)
	return hmac.Equal(gotSig, expectedSig)
}

func launchRunnerInstance(ctx context.Context, jobID int64) {
	cfg, err := config.LoadDefaultConfig(ctx)
	if err != nil {
		return
	}
	client := ec2.NewFromConfig(cfg)

	// Hardened User Data to download and configure runner
	userData := `#!/bin/bash
// snippet-3
set -euo pipefail
# hard-coded userdata sequence continues below in next snippet`

	_, err = client.RunInstances(ctx, &ec2.RunInstancesInput{
		ImageId:      ptr.String("ami-0abcdef1234567890"), // Hardened Linux AMI
		InstanceType: types.InstanceTypeT3Medium,
		MinCount:     ptr.Int32(1),
		MaxCount:     ptr.Int32(1),
		UserData:     ptr.String(userData),
		TagSpecifications: []types.TagSpecification{
			{
				ResourceType: types.ResourceTypeInstance,
				Tags: []types.Tag{
					{Key: ptr.String("github-job-id"), Value: ptr.String(string(jobID))},
					{Key: ptr.String("Role"), Value: ptr.String("ephemeral-runner")},
				},
			},
		},
	})
}

func contains(arr []string, target string) bool {
	for _, val := range arr {
		if val == target {
			return true
		}
	}
	return false
}

snippet-3.sh

#!/bin/bash
set -euo pipefail

# System preparation & updates
export DEBIAN_FRONTEND=noninteractive
apt-get update && apt-get install -y jq curl git uidmap

# Install rootless docker for container isolation
curl -fsSL https://get.docker.com -o get-docker.sh
sh get-docker.sh
runuser -l ubuntu -c "dockerd-rootless-setuptool.sh install"
export DOCKER_HOST=unix:///run/user/1000/docker.sock

# Fetch ephemeral registration token using the metadata or SSM Parameter
# For security, orchestrator should populate a parameter or write to Secrets Manager
RUNNER_VERSION="2.316.1"
cd /home/ubuntu && mkdir actions-runner && cd actions-runner
curl -o actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz -L https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz
tar xzf ./actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz
chown -R ubuntu:ubuntu /home/ubuntu/actions-runner

# Fetch token via dynamic API parameter (orchestrator handles storage)
REG_TOKEN=$(aws ssm get-parameter --name "/github/runner-token" --with-decryption --query "Parameter.Value" --output text)

# Configure the runner agent under the unprivileged user
runuser -l ubuntu -c "/home/ubuntu/actions-runner/config.sh --url https://github.com/my-org/my-hardened-app --token ${REG_TOKEN} --ephemeral --name ec2-runner-$(hostname) --unattended --replace"

# Run the runner agent. --once enforces that it runs exactly one job, then shuts down.
runuser -l ubuntu -c "/home/ubuntu/actions-runner/run.sh --once"

# Self-termination: The runner finished its job. Trigger immediate shutdown.
# An IAM role attached to the instance permits ONLY ec2:TerminateInstances on self.
INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id)
REGION=$(curl -s http://169.254.169.254/latest/meta-data/placement/region)
aws ec2 terminate-instances --instance-ids "$INSTANCE_ID" --region "$REGION"

snippet-4.yaml

name: Build and Deploy Production
on:
  push:
    branches:
      - main

jobs:
  release:
    runs-on: [self-hosted, aws-ephemeral]
    # Required permissions block to request the OIDC JWT token
    permissions:
      id-token: write
      contents: read
    
    steps:
      - name: Checkout Source Code
        uses: actions/checkout@v4

      - name: Authenticate with Vault & Fetch Dynamic Secrets
        id: vault_secrets
        uses: hashicorp/vault-action@v2.8.1
        with:
          url: https://vault.service.internal:8200
          method: jwt
          path: github-actions
          role: repo-build-role
          # Retrieve dynamic Postgres credentials from Vault database secrets engine
          secrets: |
            database/creds/app-prod-role username | DB_USER ;
            database/creds/app-prod-role password | DB_PASSWORD ;
            secret/data/production/api-keys slack_webhook | SLACK_WEBHOOK

      - name: Run Build and Database Migration
        env:
          DB_USER: ${{ steps.vault_secrets.outputs.DB_USER }}
          DB_PASSWORD: ${{ steps.vault_secrets.outputs.DB_PASSWORD }}
        run: |
          echo "Running migration on database..."
          go test -v ./...
          go run cmd/migrate/main.go

snippet-5.hcl

# Secure role mapping in HashiCorp Vault restricting JWT authentication
resource "vault_jwt_auth_backend_role" "production_build" {
  backend        = vault_jwt_auth_backend.github.path
  role_name      = "repo-build-role"
  token_policies = ["production-secrets-read"]

  # Bind audiences to prevent token misuse on arbitrary endpoints
  bound_audiences = ["https://vault.service.internal:8200"]

  # Enforce strict repository ownership and branch rules
  bound_claims = {
    # Restrict to a specific repository in your organization
    repository = "my-org/my-hardened-app"
    # Ensure this role is only assumed from the main branch
    ref = "refs/heads/main"
    # Ensure only trusted events (e.g. push or release) can assume this role
    event_name = "push"
  }

  user_claim            = "actor"
  role_type             = "jwt"
  token_ttl             = 900  # Token expires in 15 minutes
  token_max_ttl         = 1800 # Hard limit of 30 minutes
}