Envoy + Rust + SignalFX Tracing Config

custom_root_span.rs

use tracing_actix_web::TracingLogger;

use actix_web_opentelemetry::RequestMetrics;
use opentelemetry::{global, runtime::TokioCurrentThread};

use actix_http::header::{self, HeaderMap, HeaderValue};
use actix_web::dev::{ServiceRequest, ServiceResponse};
use opentelemetry_zipkin::Propagator as ZipkinPropagator;
use tracing::Span;
use tracing_actix_web::{DefaultRootSpanBuilder, RootSpanBuilder};
use tracing_bunyan_formatter::{BunyanFormattingLayer, JsonStorageLayer};
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::{EnvFilter, Registry};


use lazy_static::lazy_static;
use std::collections::BTreeMap;

lazy_static! {
    static ref OTEL_RESOURCE_ATTRIBUTES: String = std::env::var("OTEL_RESOURCE_ATTRIBUTES")
        .unwrap_or_else(|_| "deployment.environment=my-env".to_string());
    static ref OTEL_ATTRS: BTreeMap<&'static str, &'static str> = OTEL_RESOURCE_ATTRIBUTES
        .split(",")
        .collect::<Vec<&'static str>>()
        .iter()
        .map(|x| {
            x.split_once("=")
                .expect("Each pair should be split by an equals sign, =")
        })
        .collect::<BTreeMap<&'static str, &'static str>>();
}

/// We will define a custom root span builder to capture additional fields, specific
/// to our application (like `deployment.environment`), on top of the ones provided
/// by `DefaultRootSpanBuilder` out of the box.
pub struct CustomRootSpanBuilder;

impl RootSpanBuilder for CustomRootSpanBuilder {
    fn on_request_start(request: &ServiceRequest) -> Span {
        // we want to filter out the healthcheck noise
        if request.path() == "/livez" {
            Span::none()
        } else {
            // there is probably a better way to do this, perhaps baking it into the macro
            // upstream would be a good choice?
            tracing_actix_web::root_span!(
                request,
                deployment.environmet = OTEL_ATTRS["deployment.environment"]
            )
        }
    }

    fn on_request_end<B>(span: Span, outcome: &Result<ServiceResponse<B>, Error>) {
        // Capture the standard fields when the request finishes.
        DefaultRootSpanBuilder::on_request_end(span, outcome);
    }
}

envoy.yaml

# assuming envoy v3 config

static_resources:
  listeners:
  - name: trace_0
    address:
      socket_address: { address: 127.0.0.1, port_value: 19876 }
    filter_chains:
    - filters:
      - name: envoy.http_connection_manager
        typed_config:
          "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
          stat_prefix: zipkin_http
          route_config:
            name: local_route
            request_headers_to_add:
              - header: {key: X-SF-Token, value: {{ .Values.otel.token }} }
            virtual_hosts:
            - name: local_service
              domains: ["zipkin_inbound"]
              routes:
              - match: { path: "/v2/trace" }
                route: { host_rewrite_literal: ingest.us1.signalfx.com, cluster: zipkin_outbound }
          http_filters:
          - name: envoy.filters.http.router
            typed_config: {}
  - address:
      socket_address:
        address: 0.0.0.0
        port_value: 8000
    filter_chains:
    - filters:
      - name: envoy.http_connection_manager
        typed_config:
          "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
          stat_prefix: ingress_http
          generate_request_id: true
          tracing:
            custom_tags:
            - tag: "deployment.environment"
              environment:
                name: SIGNAL_FX_ENV
                default_value: "your-env"
            provider:
              name: envoy.tracers.zipkin
              typed_config:
                "@type": type.googleapis.com/envoy.config.trace.v3.ZipkinConfig
                collector_cluster: zipkin_inbound
                collector_endpoint: "/v2/trace"
                collector_endpoint_version: HTTP_JSON
            verbose: true
          route_config:
            name: local_route
            virtual_hosts:
            - name: local_service
              domains: ['*']
              routes:
              - match: { prefix: / }
                route:
                  cluster: my-service
          http_filters:
          - name: envoy.filters.http.router
            typed_config: {}
          use_remote_address: true
  clusters:
  - name: zipkin_inbound
    connect_timeout: 1s
    type: static
    load_assignment:
      cluster_name: zipkin_inbound
      endpoints:
      - lb_endpoints:
        - endpoint:
            address:
              socket_address:
                address: 127.0.0.1
                port_value: 19876

  - name: zipkin_outbound
    connect_timeout: 10s
    type: LOGICAL_DNS
    lb_policy: ROUND_ROBIN
    load_assignment:
      cluster_name: zipkin_outbound
      endpoints:
      - lb_endpoints:
        - endpoint:
            address:
              socket_address:
                address: ingest.us1.signalfx.com
                port_value: 443
    transport_socket:
      name: envoy.transport_sockets.tls
      typed_config: {}

  - name: my-service
    connect_timeout: 0.25s
    type: strict_dns
    lb_policy: round_robin
    load_assignment:
      cluster_name: my-service
      endpoints:
      - lb_endpoints:
        - endpoint:
            address:
              socket_address:
                address: my-service.svc.cluster.local
                port_value: 80
  admin:
  access_log_path: "/tmp/admin_access.log"
  address:
    socket_address:
      address: 0.0.0.0
      port_value: 8001

tracing_setup.rs

fn init_telemetry() -> Result<()> {
    use reqwest::header;
    let cb = reqwest::Client::builder();
    let mut headers = header::HeaderMap::new();
    let value = header::HeaderValue::from_str(&std::env::var("OTEL_EXPORTER_JAEGER_PASSWORD")?)?;
    headers.insert("X-SF-Token", value);
    let client = cb.default_headers(headers).build()?;

    global::set_text_map_propagator(ZipkinPropagator::new());
    let tracer = opentelemetry_zipkin::new_pipeline()
        // providing a custom client here so we can inject the required HTTP headers
        // to work with signalfx
        .with_collector_endpoint(&std::env::var("OTEL_EXPORTER_ZIPKIN_ENDPOINT")?)
        .with_http_client(client)
        .install_batch(TokioCurrentThread)?;

    // Filter based on level - trace, debug, info, warn, error
    // Tunable via `RUST_LOG` env variable
    let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("debug"));
    // Create a `tracing` layer using the Jaeger tracer
    let telemetry = tracing_opentelemetry::layer().with_tracer(tracer);
    // Create a `tracing` layer to emit spans as structured logs to stdout
    let app_name = "my-web-app";
    let formatting_layer = BunyanFormattingLayer::new(app_name.into(), std::io::stdout);
    // Combined them all together in a `tracing` subscriber
    let subscriber = Registry::default()
        .with(env_filter)
        .with(telemetry)
        .with(JsonStorageLayer)
        .with(formatting_layer);
    tracing::subscriber::set_global_default(subscriber)?;
    Ok(())
}