Automating Ephemeral TLS Session Resumption Key Rotation Across Multi-Region Envoy Proxies — code snippets

snippet-1.yaml

# Static Envoy configuration demonstrating inline, hardcoded session ticket keys.
# DO NOT USE THIS IN PRODUCTION: Rotating keys requires a full configuration reload or process restart.
static_resources:
  listeners:
  - name: ingress_edge
    address:
      socket_address:
        address: 0.0.0.0
        port_value: 443
    filter_chains:
    - filters:
      - name: envoy.filters.network.http_connection_manager
        typed_config:
          "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
          stat_prefix: ingress_http
          route_config:
            name: local_route
            virtual_hosts:
            - name: api_service
              domains: ["api.domain.com"]
              routes:
              - match: { prefix: "/" }
                route: { cluster: api_backend }
      transport_socket:
        name: envoy.transport_sockets.tls
        typed_config:
          "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext
          common_tls_context:
            tls_certificates:
            - certificate_chain: { filename: "/etc/envoy/certs/server.crt" }
              private_key: { filename: "/etc/envoy/certs/server.key" }
            session_ticket_keys:
              keys:
              # Base64 encoded 80-byte key: 16B name + 32B AES key + 32B HMAC key
              - inline_bytes: "MTIzNDU2Nzg5MDEyMzQ1NlM5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODk="

snippet-2.go

package main

import (
	"crypto/rand"
	"encoding/base64"
	"fmt"
	"io"
)

// STEK represents the 80-byte Session Ticket Encryption Key structure required by Envoy.
type STEK struct {
	Name          [16]byte // 16 bytes key name identifier
	EncryptionKey [32]byte // 32 bytes AES-256 key
	HMACKey       [32]byte // 32 bytes HMAC-SHA256 key
}

// GenerateSTEK creates a cryptographically secure 80-byte STEK.
func GenerateSTEK() (*STEK, error) {
	var stek STEK
	
	// Read cryptographically secure pseudorandom bytes for all fields
	if _, err := io.ReadFull(rand.Reader, stek.Name[:]); err != nil {
		return nil, fmt.Errorf("failed to generate key name: %w", err)
	}
	if _, err := io.ReadFull(rand.Reader, stek.EncryptionKey[:]); err != nil {
		return nil, fmt.Errorf("failed to generate encryption key: %w", err)
	}
	if _, err := io.ReadFull(rand.Reader, stek.HMACKey[:]); err != nil {
		return nil, fmt.Errorf("failed to generate HMAC key: %w", err)
	}

	return &stek, nil
}

// Bytes serializes the STEK into a single 80-byte slice.
func (s *STEK) Bytes() []byte {
	buf := make([]byte, 80)
	copy(buf[0:16], s.Name[:])
	copy(buf[16:48], s.EncryptionKey[:])
	copy(buf[48:80], s.HMACKey[:])
	return buf
}

// Base64 returns the base64 representation of the serialized 80-byte key.
func (s *STEK) Base64() string {
	return base64.StdEncoding.EncodeToString(s.Bytes())
}

snippet-3.go

package main

import (
	"context"
	"fmt"
	"sync"

	corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
	tlsv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
	sdsv3 "github.com/envoyproxy/go-control-plane/envoy/service/secret/v3"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
	"google.golang.org/protobuf/types/known/anypb"
)

type SDSServer struct {
	sdsv3.UnimplementedSecretDiscoveryServiceServer
	mu            sync.RWMutex
	currentSecret *tlsv3.Secret
	subscribers   map[string]chan *tlsv3.Secret
}

func NewSDSServer() *SDSServer {
	return &SDSServer{
		subscribers: make(map[string]chan *tlsv3.Secret),
	}
}

// UpdateSTEKKeyring updates the internal key storage and notifies all streaming clients.
func (s *SDSServer) UpdateSTEKKeyring(keyring [][]byte) error {
	s.mu.Lock()
	defer s.mu.Unlock()

	var dataSources []*corev3.DataSource
	for _, key := range keyring {
		dataSources = append(dataSources, &corev3.DataSource{
			Specifier: &corev3.DataSource_InlineBytes{
				InlineBytes: key,
			},
		})
	}

	secret := &tlsv3.Secret{
		Name: "tls_session_ticket_keys",
		Type: &tlsv3.Secret_SessionTicketKeys{
			SessionTicketKeys: &tlsv3.TlsSessionTicketKeys{
				Keys: dataSources,
			},
		},
	}

	s.currentSecret = secret
	for clientID, ch := range s.subscribers {
		// Non-blocking write to avoid blocking on slow clients
		select {
		case ch <- secret:
		default:
			fmt.Printf("Subscriber %s channel full, skipping update\n", clientID)
		}
	}
	return nil
}

func (s *SDSServer) StreamSecrets(stream sdsv3.SecretDiscoveryService_StreamSecretsServer) error {
	clientID := fmt.Sprintf("%p", stream)
	ch := make(chan *tlsv3.Secret, 1)

	s.mu.Lock()
	s.subscribers[clientID] = ch
	if s.currentSecret != nil {
		ch <- s.currentSecret
	}
	s.mu.Unlock()

	defer func() {
		s.mu.Lock()
		delete(s.subscribers, clientID)
		close(ch)
		s.mu.Unlock()
	}()

	ctx := stream.Context()
	for {
		select {
		case <-ctx.Done():
			return ctx.Err()
		case secret := <-ch:
			anySecret, err := anypb.New(secret)
			if err != nil {
				return status.Errorf(codes.Internal, "failed to serialize secret: %v", err)
			}

			response := &sdsv3.DiscoveryResponse{
				VersionInfo: fmt.Sprintf("%d", secret.GetSessionTicketKeys().Hash()), // Key tracking
				Resources:   []*anypb.Any{anySecret},
				TypeUrl:     "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret",
			}

			if err := stream.Send(response); err != nil {
				return err
			}
		}
	}
}

snippet-4.yaml

# Dynamic Envoy configuration using Secret Discovery Service (SDS) for STEK updates.
static_resources:
  listeners:
  - name: ingress_edge
    address:
      socket_address:
        address: 0.0.0.0
        port_value: 443
    filter_chains:
    - filters:
      - name: envoy.filters.network.http_connection_manager
        typed_config:
          "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
          stat_prefix: ingress_http
          route_config:
            name: local_route
            virtual_hosts:
            - name: api_service
              domains: ["api.domain.com"]
              routes:
              - match: { prefix: "/" }
                route: { cluster: api_backend }
      transport_socket:
        name: envoy.transport_sockets.tls
        typed_config:
          "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext
          common_tls_context:
            tls_certificates:
            - certificate_chain: { filename: "/etc/envoy/certs/server.crt" }
              private_key: { filename: "/etc/envoy/certs/server.key" }
            # Reference the TLS session ticket keys dynamically via SDS
            session_ticket_keys_sds_secret_config:
              name: tls_session_ticket_keys
              sds_config:
                api_config_source:
                  api_type: GRPC
                  transport_api_version: V3
                  grpc_services:
                  - envoy_grpc:
                      cluster_name: sds_control_plane
  clusters:
  - name: sds_control_plane
    type: STRICT_DNS
    lb_policy: ROUND_ROBIN
    http2_protocol_options: {}
    load_assignment:
      cluster_name: sds_control_plane
      endpoints:
      - lb_endpoints:
        - endpoint:
            address:
              socket_address:
                address: local-sds-agent.internal
                port_value: 8089
    # Secure the SDS channel using mutual TLS
    transport_socket:
      name: envoy.transport_sockets.tls
      typed_config:
        "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
        common_tls_context:
          tls_certificates:
          - certificate_chain: { filename: "/etc/envoy/certs/sds-client.crt" }
            private_key: { filename: "/etc/envoy/certs/sds-client.key" }
          validation_context:
            trusted_ca: { filename: "/etc/envoy/certs/sds-ca.crt" }

snippet-5.go

package main

import (
	"context"
	"fmt"
)

type KeyringManager struct {
	sdsServer   *SDSServer
	storagePath string // e.g., local state, Consul, or Vault path
}

func NewKeyringManager(server *SDSServer, storage string) *KeyringManager {
	return &KeyringManager{
		sdsServer:   server,
		storagePath: storage,
	}
}

// RotateKeyring executes a rotation step. It retrieves existing keys, demotes them,
// generates a new active key, and updates the regional SDS servers.
func (km *KeyringManager) RotateKeyring(ctx context.Context) error {
	// 1. Fetch current keyring from distributed store (simulated here)
	existingKeyring, err := km.loadKeyringFromVault(ctx)
	if err != nil {
		return fmt.Errorf("failed to load keyring: %w", err)
	}

	// 2. Generate new active key
	newActiveKey, err := GenerateSTEK()
	if err != nil {
		return fmt.Errorf("failed to generate new key: %w", err)
	}

	// 3. Construct new keyring: [NewActiveKey, OldActiveKey, OlderActiveKey]
	// We keep a maximum of 3 keys (Active, Accept-Only T-1, Accept-Only T-2)
	newKeyring := [][]byte{newActiveKey.Bytes()}
	
	// Keep up to 2 historical keys (creating a 36-hour grace period if rotated every 12 hours)
	for i := 0; i < len(existingKeyring) && i < 2; i++ {
		newKeyring = append(newKeyring, existingKeyring[i])
	}

	// 4. Save new keyring back to Vault for global consistency
	if err := km.saveKeyringToVault(ctx, newKeyring); err != nil {
		return fmt.Errorf("failed to persist rotated keyring: %w", err)
	}

	// 5. Update the local gRPC SDS Server memory state to push to Envoy instances immediately
	if err := km.sdsServer.UpdateSTEKKeyring(newKeyring); err != nil {
		return fmt.Errorf("failed to update SDS server keyring: %w", err)
	}

	return nil
}

func (km *KeyringManager) loadKeyringFromVault(ctx context.Context) ([][]byte, error) {
	// In production, use the HashiCorp Vault Go client: vault.NewClient()
	// This mock returns an empty keyring or mock keys for simulation
	return [][]byte{}, nil
}

func (km *KeyringManager) saveKeyringToVault(ctx context.Context, keyring [][]byte) error {
	// Write serialized keyring to secret/data/ingress/tls_stek
	return nil
}

snippet-6.yaml

# Prometheus Alerting Rule to detect TLS Session Resumption degradation.
groups:
- name: tls_resumption_alerts
  rules:
  - alert: TLSSessionResumptionRateLow
    expr: |
      sum(rate(envoy_listener_admin_http_ssl_session_reused_total[5m])) by (region)
      /
      sum(rate(envoy_listener_admin_http_ssl_handshake_total[5m])) by (region)
      * 100 < 30
    for: 10m
    labels:
      severity: warning
      team: devsecops
    annotations:
      summary: "Low TLS session resumption rate in region {{$labels.region}}"
      description: "The TLS session resumption rate has dropped below 30% for over 10 minutes in region {{$labels.region}}. This may indicate out-of-sync STEK keyrings or SDS delivery failures."