Alloy OTEL

compose.yaml

services:
  nats:
    image: nats
    ports:
      - 4222:4222
      - 6222:6222
      - 8222:8222
  alloy:
    image: grafana/alloy
    command:
      - "run"
      - "--server.http.listen-addr=0.0.0.0:12345"
      - "/etc/alloy/config.alloy"
    ports:
      - 12345:12345
      - 4317:4317
      - 4318:4318
    environment:
      - GRAFANA_CLOUD_OTEL_ENDPOINT=${GRAFANA_CLOUD_OTEL_ENDPOINT}
      - GRAFANA_CLOUD_OTEL_USERNAME=${GRAFANA_CLOUD_OTEL_USERNAME}
      - GRAFANA_CLOUD_OTEL_PASSWORD=${GRAFANA_CLOUD_OTEL_PASSWORD}
    configs:
      - source: config.alloy
        target: /etc/alloy/config.alloy
  node-exporter:
    image: prom/node-exporter
    ports:
      - 9100:9100
    volumes:
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
      - /:/rootfs:ro
    command:
      - --path.procfs=/host/proc
      - --path.sysfs=/host/sys
      - --path.rootfs=/rootfs
      - --collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)

configs:
  config.alloy:
    content: |
      // Allow us to receive logs, metrics, and traces via OTLP.
      otelcol.receiver.otlp "default" {
        // Configures the default grpc endpoint "0.0.0.0:4317".
        grpc { }
        // Configures the default http/protobuf endpoint "0.0.0.0:4318".
        http { }

        output {
          logs    = [otelcol.processor.resourcedetection.default.input]
          metrics = [otelcol.processor.resourcedetection.default.input]
          traces  = [otelcol.processor.resourcedetection.default.input]
        }
      }

      // Detect resource attributes.
      otelcol.processor.resourcedetection "default" {
        // If you want cloud resource detection, add
        // "gcp", "ec2", "ecs", "elastic_beanstalk", "eks",
        // "lambda", "azure", "aks", "consul", "heroku".
        detectors = ["env", "system"]

        system {
          hostname_sources = ["os"]
        }

        output {
          logs    = [otelcol.processor.transform.drop_unneeded_resource_attributes.input]
          metrics = [otelcol.processor.transform.drop_unneeded_resource_attributes.input]
          traces  = [otelcol.processor.transform.drop_unneeded_resource_attributes.input]
        }
      }

      // Drop unneded resource attributes.
      otelcol.processor.transform "drop_unneeded_resource_attributes" {
        error_mode = "ignore"

        trace_statements {
          context    = "resource"
          statements = [
            `delete_key(attributes, "k8s.pod.start_time")`,
            `delete_key(attributes, "os.description")`,
            `delete_key(attributes, "os.type")`,
            `delete_key(attributes, "process.command_args")`,
            `delete_key(attributes, "process.executable.path")`,
            `delete_key(attributes, "process.pid")`,
            `delete_key(attributes, "process.runtime.description")`,
            `delete_key(attributes, "process.runtime.name")`,
            `delete_key(attributes, "process.runtime.version")`,
          ]
        }

        metric_statements {
          context    = "resource"
          statements = [
            `delete_key(attributes, "k8s.pod.start_time")`,
            `delete_key(attributes, "os.description")`,
            `delete_key(attributes, "os.type")`,
            `delete_key(attributes, "process.command_args")`,
            `delete_key(attributes, "process.executable.path")`,
            `delete_key(attributes, "process.pid")`,
            `delete_key(attributes, "process.runtime.description")`,
            `delete_key(attributes, "process.runtime.name")`,
            `delete_key(attributes, "process.runtime.version")`,
          ]
        }

        log_statements {
          context    = "resource"
          statements = [
            `delete_key(attributes, "k8s.pod.start_time")`,
            `delete_key(attributes, "os.description")`,
            `delete_key(attributes, "os.type")`,
            `delete_key(attributes, "process.command_args")`,
            `delete_key(attributes, "process.executable.path")`,
            `delete_key(attributes, "process.pid")`,
            `delete_key(attributes, "process.runtime.description")`,
            `delete_key(attributes, "process.runtime.name")`,
            `delete_key(attributes, "process.runtime.version")`,
          ]
        }

        output {
          logs    = [
            otelcol.processor.batch.default.input,
          ]
          metrics = [
            otelcol.processor.transform.add_resource_attributes_as_metric_attributes.input,
          ]
          traces  = [
            otelcol.processor.tail_sampling.default.input,
            otelcol.connector.host_info.usage_metrics.input,
          ]
        }
      }

      // Accept spans to generate usage metrics.
      otelcol.connector.host_info "usage_metrics" {
        host_identifiers = ["host.name"]

        output {
          metrics = [otelcol.processor.batch.default.input]
        }
      }

      // Add resource attributes as metric attributes.
      otelcol.processor.transform "add_resource_attributes_as_metric_attributes" {
        error_mode = "ignore"

        metric_statements {
          context    = "datapoint"
          statements = [
            `set(attributes["deployment.environment"], resource.attributes["deployment.environment"])`,
            `set(attributes["service.version"], resource.attributes["service.version"])`,
          ]
        }

        output {
          metrics = [otelcol.processor.filter.default.input]
        }
      }

      // Filter all metrics that are not part of the following services.
      // - findata
      otelcol.processor.filter "default" {
        error_mode = "ignore"

        metrics {
          metric = [
            `not IsMatch(resource.attributes["service.name"], "findata")`,
          ]
        }

        output {
          metrics = [otelcol.processor.batch.default.input]
        }
      }

      // Sample traces to reduce the amount of data sent to the backend.
      otelcol.processor.tail_sampling "default" {
        // Select all traces that have an error or unset status code.
        policy {
          name = "error_traces"
          type = "status_code"

          status_code {
            status_codes = ["ERROR", "UNSET"]
          }
        }

        // Select all traces that take longer than 1000ms.
        policy {
          name = "high_latency_traces"
          type = "latency"

          latency {
            threshold_ms = 1000
          }
        }

        // Sample 10% of successful traces.
        policy {
          name = "successful_traces"
          type = "probabilistic"

          probabilistic {
            sampling_percentage = 10.0
          }
        }

        output {
          traces  = [otelcol.processor.batch.default.input]
        }
      }

      otelcol.processor.batch "default" {
        output {
          logs    = [otelcol.exporter.otlphttp.grafana_cloud.input]
          metrics = [otelcol.exporter.otlphttp.grafana_cloud.input]
          traces  = [otelcol.exporter.otlphttp.grafana_cloud.input]
        }
      }

      otelcol.exporter.otlphttp "grafana_cloud" {
        client {
          endpoint = sys.env("GRAFANA_CLOUD_OTEL_ENDPOINT")
          auth     = otelcol.auth.basic.grafana_cloud.handler
        }
      }

      otelcol.auth.basic "grafana_cloud" {
        username = sys.env("GRAFANA_CLOUD_OTEL_USERNAME")
        password = sys.env("GRAFANA_CLOUD_OTEL_PASSWORD")
      }