otel-config.yaml

receivers:
  hostmetrics:
    collection_interval: 15s
    scrapers:
      cpu:
      memory:
        metrics:
          system.memory.utilization:
            enabled: true
      load:
        cpu_average: true
      network:
      processes:

  # Requires root user right now, will investigate more later
  hostmetrics/process:
    collection_interval: 30s
    scrapers:
      process:

  hostmetrics/disk:
    collection_interval: 1m
    scrapers:
      disk:
      filesystem:
        metrics:
          system.filesystem.utilization:
            enabled: true
      paging:

processors:
  batch: # https://github.com/open-telemetry/opentelemetry-collector/blob/main/processor/batchprocessor/README.md
    timeout: 30s

  # will pull k/v from OTEL_RESOURCE_ATTRIBUTES. Format <key1>=<value1>,<key2>=<value2>,...
  resourcedetection/env:
    detectors: ["env"]
    timeout: 2s
    override: false

  resourcedetection/system:
    detectors: ["system"]
    system:
      hostname_sources: ["os"]

  resourcedetection/ec2:
    detectors: ["ec2"]
    ec2:
      tags:
        - role # Ex. nomad-client, nomad-server, consul
        - cluster # Ex. market, idig
        - workload # Ex. gpu, cpu, memory, spot
        - nomad-clients # existing tag, can remove later
  metricstransform:
    transforms:
      - include: system.memory.usage
        experimental_match_labels: { state: "used" }
        action: insert
        new_name: "system.memory.usage.used"
      - include: system.memory.usage
        experimental_match_labels: { state: "free" }
        action: insert
        new_name: "system.memory.usage.free"
      - include: system.memory.usage
        experimental_match_labels: { state: "buffered" }
        action: insert
        new_name: "system.memory.usage.buffered"
      - include: system.memory.usage
        experimental_match_labels: { state: "cached" }
        action: insert
        new_name: "system.memory.usage.cached"
      - include: system.memory.usage
        experimental_match_labels: { state: "slab_reclaimable" }
        action: insert
        new_name: "system.memory.usage.slab_reclaimable"
      - include: system.memory.usage
        experimental_match_labels: { state: "slab_unreclaimable" }
        action: insert
        new_name: "system.memory.usage.slab_unreclaimable"
      - include: system.cpu.time
        experimental_match_labels: { state: "idle" }
        action: insert
        new_name: "system.cpu.time.idle"
      - include: system.cpu.time
        experimental_match_labels: { state: "interrupt" }
        action: insert
        new_name: "system.cpu.time.interrupt"
      - include: system.cpu.time
        experimental_match_labels: { state: "nice" }
        action: insert
        new_name: "system.cpu.time.nice"
      - include: system.cpu.time
        experimental_match_labels: { state: "softirq" }
        action: insert
        new_name: "system.cpu.time.softirq"
      - include: system.cpu.time
        experimental_match_labels: { state: "steal" }
        action: insert
        new_name: "system.cpu.time.steal"
      - include: system.cpu.time
        experimental_match_labels: { state: "system" }
        action: insert
        new_name: "system.cpu.time.system"
      - include: system.cpu.time
        experimental_match_labels: { state: "user" }
        action: insert
        new_name: "system.cpu.time.user"
      - include: system.cpu.time
        experimental_match_labels: { state: "wait" }
        action: insert
        new_name: "system.cpu.time.wait"
  filter:
    metrics:
      exclude:
        match_type: strict
        metric_names:
          - system.memory.usage
          - system.cpu.time

extensions:
  pprof: # Enable pprof for profiling if performance issues arise. https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/extension/pprofextension
  memory_ballast: # https://github.com/open-telemetry/opentelemetry-collector/blob/main/extension/ballastextension/README.md
    size_mib: 64
  zpages: # Enable for zPages Debugging, https://github.com/open-telemetry/opentelemetry-collector/blob/main/extension/zpagesextension/README.md

exporters:
  otlp:
    endpoint: api.honeycomb.io:443
    # Hosts will only send metrics, would prefer snappy but not available yet
    compression: gzip
    headers:
      x-honeycomb-team: "${HONEYCOMB_APIKEY}"
      x-honeycomb-dataset: infra

service:
  extensions: [memory_ballast]
  pipelines:
    metrics:
      receivers: [hostmetrics, hostmetrics/disk]
      processors: [
          resourcedetection/env,
          resourcedetection/system,
          resourcedetection/ec2,
          batch,
          metricstransform,
          filter,
        ] # There's a recommended order for processors, https://github.com/open-telemetry/opentelemetry-collector/blob/main/processor/README.md
      exporters: [otlp]