README.md

• Create a Shikari Cluster with nomad-consul-secure scenario

  cd scenarios/nomad-consul-secure
  shikari create -n apigw -s 1 -c 1 -i ~/.shikari/c-1.20.1-n-1.9.3-v-1.18.2-b-0.18.1-fedora/hashibox.qcow2
  

• Export the environment variables

  eval $(shikari env -n apigw consul,nomad -ta) 
  

• Run the OTel stack

  nomad run tempo.nomad.hcl
  nomad run otel-collector.nomad.hcl
  nomad run grafana.nomad.hcl
  

• Fetch the endpoint address of otel-collector allocation and update the address in proxy-defaults

  nomad job allocs otel-collector
  ID        Node ID   Task Group      Version  Desired  Status   Created     Modified
  e8ea2042  effcce5b  otel-collector  1        run      running  33m49s ago  15m1s ago
  
  nomad status e8e  | grep ^\*otel-collector
  *otel-collector  yes      192.168.105.109:4317
  

• Update proxy-defaults.hcl

• write the config entry

  consul config write proxy-defaults.hcl
  

• Clone the git repository to run api-gateway and example job

      git clone https://github.com/hashicorp-guides/consul-api-gateway-on-nomad
  
     cd consul-api-gateway-on-nomad
  
     nomad namespace apply -description "namespace for Consul API Gateways" ingress
  
     consul acl binding-rule create \
       -method 'nomad-workloads' \
       -description 'Nomad API gateway' \
       -bind-type 'templated-policy' \
       -bind-name 'builtin/api-gateway' \
       -bind-vars 'Name=${value.nomad_job_id}' \
       -selector '"nomad_service" not in value and value.nomad_namespace==ingress'
  
     nomad var put -namespace ingress \
       nomad/jobs/my-api-gateway/gateway/setup \
       consul_cacert=@$CONSUL_CACERT
  
     # this scenario doesn't configure client cert auth for agents HTTPs. So remove CERT_FILE and CERT_KEY related variables and template block
     sed -e '85,105d;69,70d' api-gateway.nomad.hcl | nomad job run -
  
  
     # Setup Hello example application
  
     consul config write example/hello-app-intentions.hcl
     consul config write example/gateway-listeners.hcl
     consul config write example/my-http-route.hcl
     nomad run example/hello-app.nomad.hcl
  

  • Access the app via API Gateway and observe the otel-collector logs

  curl 192.168.105.109:8088/hello -H "x-client-trace-id: 1"
  Hello World!
  

  nomad logs -stderr -f -job otel-collector
  2024-11-26T02:23:57.133Z        info    TracesExporter  {"kind": "exporter", "data_type": "traces", "name": "debug", "resource spans": 1, "spans": 1}
  2024-11-26T02:23:57.133Z        info    ResourceSpans #0
  Resource SchemaURL:
  Resource attributes:
       -> service.name: Str(unknown_service:envoy)
  ScopeSpans #0
  ScopeSpans SchemaURL:
  InstrumentationScope
  Span #0
      Trace ID       : 40b796105dd5a8c384f5d6d05ab6bf2e
      Parent ID      : 5a5b81a6b2e6beb2
      ID             : 272b382408dc99d2
      Name           : ingress
      Kind           : Server
      Start time     : 2024-11-26 02:23:56.739748 +0000 UTC
      End time       : 2024-11-26 02:23:56.740423 +0000 UTC
      Status code    : Unset
      Status message :
  Attributes:
       -> node_id: Str(_nomad-task-575c4855-e359-05a9-6690-4aee71c97756-group-apps-hello-app-9090-sidecar-proxy)
       -> zone: Str()
       -> guid:x-request-id: Str(76bf6e49-a8e5-be0e-bf5c-2a2605b7f739)
       -> http.url: Str(http://192.168.105.109:8088/hello)
       -> http.method: Str(GET)
       -> downstream_cluster: Str(-)
       -> user_agent: Str(curl/8.7.1)
       -> http.protocol: Str(HTTP/1.1)
       -> peer.address: Str(172.26.64.1)
       -> guid:x-client-trace-id: Str(1)
       -> request_size: Str(0)
       -> response_size: Str(13)
       -> component: Str(proxy)
       -> upstream_cluster: Str(local_app)
       -> upstream_cluster.name: Str(local_app)
       -> http.status_code: Str(200)
       -> response_flags: Str(-)
          {"kind": "exporter", "data_type": "traces", "name": "debug"}
  2024-11-26T02:23:57.508Z        info    TracesExporter  {"kind": "exporter", "data_type": "traces", "name": "debug", "resource spans": 1, "spans": 1}
  2024-11-26T02:23:57.508Z        info    ResourceSpans #0
  Resource SchemaURL:
  Resource attributes:
       -> service.name: Str(unknown_service:envoy)
  ScopeSpans #0
  ScopeSpans SchemaURL:
  InstrumentationScope
  Span #0
      Trace ID       : 40b796105dd5a8c384f5d6d05ab6bf2e
      Parent ID      :
      ID             : 5a5b81a6b2e6beb2
      Name           : egress 192.168.105.109:8088
      Kind           : Client
      Start time     : 2024-11-26 02:23:56.739334 +0000 UTC
      End time       : 2024-11-26 02:23:56.740683 +0000 UTC
      Status code    : Unset
      Status message :
  Attributes:
       -> node_id: Str(my-api-gateway)
       -> zone: Str()
       -> guid:x-request-id: Str(76bf6e49-a8e5-be0e-bf5c-2a2605b7f739)
       -> http.url: Str(http://192.168.105.109:8088/hello)
       -> http.method: Str(GET)
       -> downstream_cluster: Str(-)
       -> user_agent: Str(curl/8.7.1)
       -> http.protocol: Str(HTTP/1.1)
       -> peer.address: Str(192.168.105.1)
       -> guid:x-client-trace-id: Str(1)
       -> request_size: Str(0)
       -> response_size: Str(13)
       -> component: Str(proxy)
       -> upstream_cluster: Str(hello-app.default.apigw.internal.08a7bf12-c6f6-6841-f73b-ab7fbd8cfdaa.consul)
       -> upstream_cluster.name: Str(hello-app.default.apigw.internal.08a7bf12-c6f6-6841-f73b-ab7fbd8cfdaa.consul)
       -> http.status_code: Str(200)
       -> response_flags: Str(-)
          {"kind": "exporter", "data_type": "traces", "name": "debug"}
  

grafana.nomad.hcl

job "grafana" {

  group "grafana" {
    network {
      port "grafana" {
        static = 3000
      }
    }

    task "grafana" {
      driver = "docker"

      config {
        image          = "grafana/grafana:10.1.1"
        ports          = ["grafana"]
        auth_soft_fail = true

        mounts = [
          {
            type   = "bind"
            source = "local/grafana-datasource.yaml"
            target = "/etc/grafana/provisioning/datasources/datasources.yaml"
          },
        ]
      }

      template {
        data        = <<EOF
datasources:
- name: Tempo
  type: tempo
  access: proxy
  orgId: 1
  url: http://{{ range nomadService "tempo" }} {{ .Address}}:3200 {{end}}
  basicAuth: false
  isDefault: true
  version: 1
  editable: false
  apiVersion: 1
  uid: tempo
  jsonData:
    httpMethod: GET
    serviceMap:
      datasourceUid: prometheus
		EOF      
        destination = "local/grafana-datasource.yaml"
      }

      identity {
        env  = true
        file = true
      }
      env {
        GF_AUTH_ANONYMOUS_ENABLED  = true
        GF_AUTH_ANONYMOUS_ORG_ROLE = Admin
        GF_AUTH_DISABLE_LOGIN_FORM = true
        GF_FEATURE_TOGGLES_ENABLE  = traceqlEditor
      }


      resources {
        cpu    = 500
        memory = 256
      }
    }
  }
}

otel-collector.nomad.hcl

job "otel-collector" {

  group "otel-collector" {
    network {
      port "otel-collector" {
        static = 4317
      }
    }
    
    service {
      name = "otel-collector"
      provider = "nomad"
      port = "otel-collector"
    }

    task "otel-collector" {
      driver = "docker"

      config {
        image          = "otel/opentelemetry-collector:0.86.0"
        ports          = ["otel-collector"]
        auth_soft_fail = true
        command =  "--config=local/otel-collector.yaml"
      }

      template {
		data = <<EOF
receivers:
  otlp:
    protocols:
      grpc:
      http:
  zipkin:

exporters:
  debug:
    verbosity: detailed
  otlp:
    endpoint: {{ range nomadService "tempo" }} {{ .Address}}:{{.Port}} {{end}}
    tls:
      insecure: true

service:
  telemetry:
    logs:
      level: debug
      
  pipelines:
    traces:
      receivers: [zipkin, otlp]
      exporters: [debug, otlp]
		EOF      
		destination = "local/otel-collector.yaml"
      }

      identity {
        env  = true
        file = true
      }

      resources {
        cpu    = 500
        memory = 256
      }
    }
  }
}

proxy-defaults.hcl

Kind = "proxy-defaults"
Name = "global"
Config {
  protocol = "http"
  envoy_extra_static_clusters_json = <<EOF
{
  "connect_timeout": "3.000s",
  "dns_lookup_family": "V4_ONLY",
  "lb_policy": "ROUND_ROBIN",
  "typed_extension_protocol_options": {
    "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": {
      "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions",
      "explicit_http_config": {
        "http2_protocol_options": {}
      }
    }
  },
  "load_assignment": {
    "cluster_name": "tempo-otlp",
    "endpoints": [
      {
        "lb_endpoints": [
          {
            "endpoint": {
              "address": {
                "socket_address": {
                  "address": "192.168.105.109",
                  "port_value": 4317
                }
              }
            }
          }
        ]
      }
    ]
  },
  "name": "tempo-otlp",
  "type": "STRICT_DNS"
}
EOF
  envoy_tracing_json               = <<EOF
{
  "http": {
    "name": "envoy.tracers.opentelemetry",
    "typedConfig": {
      "@type": "type.googleapis.com/envoy.config.trace.v3.OpenTelemetryConfig",
      "grpc_service": {
        "envoy_grpc": {
          "cluster_name": "tempo-otlp"
        }
      }
    }
  }
}
EOF
}

tempo.nomad.hcl

job "tempo" {

  group "tempo" {
    network {
      port "otlp_grpc" {
        to = 4317
      }
    }

    service {
      name     = "tempo"
      provider = "nomad"
      port     = "otlp_grpc"
    }

    task "tempo" {

      driver = "docker"

      config {
        image          = "grafana/tempo:latest"
        ports          = ["otlp_grpc"]
        auth_soft_fail = true
        command        = "-config.file=local/tempo.yaml"
      }

      template {
        data        = <<EOF
server:
  http_listen_port: 3200
  log_level: debug 

query_frontend:
  search:
    duration_slo: 5s
    throughput_bytes_slo: 1.073741824e+09
  trace_by_id:
    duration_slo: 5s

distributor:
  receivers:                           # this configuration will listen on all ports and protocols that tempo is capable of.
    jaeger:                            # the receives all come from the OpenTelemetry collector.  more configuration information can
      protocols:                       # be found there: https://github.com/open-telemetry/opentelemetry-collector/tree/main/receiver
        thrift_http:                   #
        grpc:                          # for a production deployment you should only enable the receivers you need!
        thrift_binary:
        thrift_compact:
    zipkin:
    otlp:
      protocols:
        http:
        grpc:
    opencensus:

ingester:
  max_block_duration: 5m               # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally

compactor:
  compaction:
    block_retention: 1h                # overall Tempo trace retention. set for demo purposes

      #metrics_generator:
      #  registry:
      #    external_labels:
      #      source: tempo
      #      cluster: docker-compose
      #  storage:
      #    path: /tmp/tempo/generator/wal
      #    remote_write:
      #      - url: http://prometheus:9090/api/v1/write
      #        send_exemplars: true

storage:
  trace:
    backend: local                     # backend configuration to use
    wal:
      path: /tmp/tempo/wal             # where to store the the wal locally
    local:
      path: /tmp/tempo/blocks

overrides:
  defaults:
    metrics_generator:
      processors: [service-graphs, span-metrics] # enables metrics generator
		EOF      
        destination = "local/tempo.yaml"
      }

      identity {
        env  = true
        file = true
      }

      resources {
        cpu    = 500
        memory = 256
      }
    }
  }
}