Consul OpenTelemetry Tracing Example

consul.hcl

# Start Consul using the below command
# consul agent -dev -config-file consul.hcl

config_entries {
  bootstrap {
    Kind = "proxy-defaults"
    Name = "global"
    Config = {
      protocol = "http"

      envoy_extra_static_clusters_json = <<EOF
    {
        "connect_timeout": "3.000s",
        "dns_lookup_family": "V4_ONLY",
        "lb_policy": "ROUND_ROBIN",
        "typed_extension_protocol_options": {
          "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": {
            "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions",
            "explicit_http_config": {
              "http2_protocol_options" : {}
            }
          }
        },
        "load_assignment": {
          "cluster_name": "otel-collector",
          "endpoints": [
            {
              "lb_endpoints": [
                {
                  "endpoint": {
                    "address": {
                      "socket_address": {
                        "address": "localhost",
                        "port_value": 4317
                      }
                    }
                  }
                }
              ]
            }
          ]
        },
        "name": "otel-collector",
        "type": "STRICT_DNS"
      }
    EOF


      envoy_tracing_json = <<EOF
    {
        "http": {
          "name": "envoy.tracers.opentelemetry",
          "typedConfig":  {
            "@type": "type.googleapis.com/envoy.config.trace.v3.OpenTelemetryConfig",
            "grpc_service": {
              "envoy_grpc": {
                "cluster_name": "otel-collector"
              }
            }
          }
        }
      }
    EOF
    }
  }
  bootstrap {
    Kind = "ingress-gateway"
    Name = "ingress-gateway"

    Listeners = [
      {
        Port     = 8080
        Protocol = "http"
        Services = [
          {
            Name = "frontend"
          }
        ]
      }
    ]
  }
}

services {
  name    = "frontend"
  port    = 9002
  address = "127.0.0.1"

  connect {
    sidecar_service {
      proxy {
        upstreams = [
          {
            destination_name = "frontend"
            local_bind_port  = 5000
          },
        ]
      }
    }
  }

  check {
    id       = "frontend-check"
    http     = "http://localhost:9002/health"
    method   = "GET"
    interval = "1s"
    timeout  = "1s"
  }
}

services {
  name    = "backend"
  id      = "backend"
  port    = 9003
  address = "127.0.0.1"

  connect {
    sidecar_service {}
  }

  check {
    id       = "backend-check"
    http     = "http://localhost:9003/health"
    method   = "GET"
    interval = "1s"
    timeout  = "1s"
  }
}

instructions.md

Start the frontend and backend application (optionally configure otel endpoints). We are using fakeservice here.

LISTEN_ADDR=127.0.0.1:9002 UPSTREAM_URIS=http://localhost:5000 NAME=frontend fake-service
LISTEN_ADDR=127.0.0.1:9003 NAME=backend fake-service

Start the sidecar proxies and the ingress gateway:

consul connect envoy -gateway ingress -register --admin-bind localhost:19002 -- -l debug
consul connect envoy -sidecar-for frontend -- -l debug
consul connect envoy -sidecar-for backend -admin-bind localhost:19001 -- -l debug

Try hitting the Ingress Gateway to access the frontend service and observe the tracese populated in the collector

curl -H "Host: frontend.ingress.example.com" -H "x-client-trace-id: 1" http://localhost:8080

Logs from the otel-collector

2023-12-14T15:18:13.850+1100    info    TracesExporter  {"kind": "exporter", "data_type": "traces", "name": "debug", "resource spans": 1, "spans": 1}
2023-12-14T15:18:13.850+1100    info    ResourceSpans #0
Resource SchemaURL:
Resource attributes:
     -> service.name: Str(unknown_service:envoy)
ScopeSpans #0
ScopeSpans SchemaURL:
InstrumentationScope
Span #0
    Trace ID       : 17c62b9984e7ed57e502b39aae0d43aa
    Parent ID      :
    ID             : 6bdeb98ba1baeacf
    Name           : egress frontend.ingress.example.com
    Kind           : Client
    Start time     : 2023-12-14 04:18:12.024349 +0000 UTC
    End time       : 2023-12-14 04:18:12.043732 +0000 UTC
    Status code    : Unset
    Status message :
Attributes:
     -> node_id: Str(ingress-gateway)
     -> zone: Str()
     -> guid:x-request-id: Str(0fd05cb5-5329-baea-9c34-d0f05996d719)
     -> http.url: Str(http://frontend.ingress.example.com/)
     -> http.method: Str(GET)
     -> downstream_cluster: Str(-)
     -> user_agent: Str(curl/7.81.0)
     -> http.protocol: Str(HTTP/1.1)
     -> peer.address: Str(127.0.0.1)
     -> guid:x-client-trace-id: Str(1)
     -> request_size: Str(0)
     -> response_size: Str(2617)
     -> component: Str(proxy)
     -> upstream_cluster: Str(frontend.default.dc1.internal.54c1fbae-747a-ad78-1835-e02f31d20700.consul)
     -> upstream_cluster.name: Str(frontend.default.dc1.internal.54c1fbae-747a-ad78-1835-e02f31d20700.consul)
     -> http.status_code: Str(200)
     -> response_flags: Str(-)
        {"kind": "exporter", "data_type": "traces", "name": "debug"}
2023-12-14T15:18:16.056+1100    info    TracesExporter  {"kind": "exporter", "data_type": "traces", "name": "debug", "resource spans": 1, "spans": 1}
2023-12-14T15:18:16.056+1100    info    ResourceSpans #0
Resource SchemaURL:
Resource attributes:
     -> service.name: Str(unknown_service:envoy)
ScopeSpans #0
ScopeSpans SchemaURL:
InstrumentationScope
Span #0
    Trace ID       : 17c62b9984e7ed57e502b39aae0d43aa
    Parent ID      : 6bdeb98ba1baeacf
    ID             : b566352b3001c237
    Name           : ingress
    Kind           : Server
    Start time     : 2023-12-14 04:18:12.029061 +0000 UTC
    End time       : 2023-12-14 04:18:12.039071 +0000 UTC
    Status code    : Unset
    Status message :
Attributes:
     -> node_id: Str(frontend-sidecar-proxy)
     -> zone: Str()
     -> guid:x-request-id: Str(0fd05cb5-5329-baea-9c34-d0f05996d719)
     -> http.url: Str(http://frontend.ingress.example.com/)
     -> http.method: Str(GET)
     -> downstream_cluster: Str(-)
     -> user_agent: Str(curl/7.81.0)
     -> http.protocol: Str(HTTP/1.1)
     -> peer.address: Str(127.0.0.1)
     -> guid:x-client-trace-id: Str(1)
     -> request_size: Str(0)
     -> response_size: Str(2617)
     -> component: Str(proxy)
     -> upstream_cluster: Str(local_app)
     -> upstream_cluster.name: Str(local_app)
     -> http.status_code: Str(200)
     -> response_flags: Str(-)
        {"kind": "exporter", "data_type": "traces", "name": "debug"}

otel-config.yaml

# Start otel-collector using the below command
# otelcol -config otel-config.yaml
extensions:
  zpages:
    endpoint: 0.0.0.0:55679

receivers:
  otlp:
    protocols:
      grpc:
      http:

exporters:
  debug:
    verbosity: detailed

service:
  pipelines:
    traces:
      receivers: [otlp]
      exporters: [debug]
  extensions: [zpages]