OTEL for Logscale

_setup.sh

helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts
helm upgrade --install --create-namespace --namespace otel-operator \
    -f .kind/otel-operator-values.yaml \
    --version ${V_OTEL_OPERATOR:-0.40.0} \
    opentelemetry-operator open-telemetry/opentelemetry-operator --wait
helm upgrade --install --namespace otel-operator -f agent-values.yaml agent  open-telemetry/opentelemetry-collector
helm upgrade --install --namespace otel-operator -f gateway-values.yaml gateway  open-telemetry/opentelemetry-collector
helm upgrade --install --namespace otel-operator -f cluster-values.yaml cluster  open-telemetry/opentelemetry-collector

agent-values.yaml

mode: daemonset
image:
  # If you want to use the core image `otel/opentelemetry-collector`, you also need to change `command.name` value to `otelcol`.
  repository: ghcr.io/segateway/containers/container-open-telemetry-collector-debian
  pullPolicy: IfNotPresent
  # Overrides the image tag whose default is the chart appVersion.
  tag: "0.87.0-otelcol-contrib-journald-1.0.0"
command:
  name: bin/otelcol-contrib
presets:
  logsCollection:
    enabled: true
    storeCheckpoints: true
    includeCollectorLogs: true
  hostMetrics:
    enabled: false
  kubernetesAttributes:
    enabled: true
    extractAllPodLabels: true
  kubeletMetrics:
    enabled: false

resources:
  limits:
    cpu: 100m
    memory: 200M

tolerations:
  - key: "CriticalAddonsOnly"
    operator: "Exists"

useGOMEMLIMIT: true

extraVolumes:
  - name: varlogjournal
    hostPath:
      path: /var/log/journal
      type: Directory
extraVolumeMounts:
  - name: varlogjournal
    mountPath: /var/log/journal

config:
  receivers:
    journald:
      directory: /var/log/journal
      priority: info
      storage: file_storage
  exporters:
    otlp/agg:
      endpoint: "http://gateway-opentelemetry-collector:4317"
      tls:
        insecure: true
  processors:
    resourcedetection/azure:
      detectors:
        - azure
      override: false
    resourcedetection/aks:
      detectors:
        - aks
      override: false
    resourcedetection/k8snode:
      detectors:
        - k8snode
      override: false
    resourcedetection/system:
      detectors:
        - system
      override: false
    k8sattributes:
      extract:
        labels:
          - from: pod
            key_regex: (.*)
            tag_name: k8s.labels.$$1
        metadata:
          - k8s.namespace.name
          - k8s.deployment.name
          - k8s.statefulset.name
          - k8s.daemonset.name
          - k8s.cronjob.name
          - k8s.job.name
          - k8s.node.name
          - k8s.pod.name
          - k8s.pod.uid
          - k8s.pod.start_time
          - k8s.container.name
          - container.image.name
          - container.image.tag
          - container.id
      filter:
        node_from_env_var: K8S_NODE_NAME
      passthrough: false
      pod_association:
        - sources:
            - from: resource_attribute
              name: k8s.pod.ip
        - sources:
            - from: resource_attribute
              name: k8s.pod.uid
        - sources:
            - from: connection
  service:
    pipelines:
      logs:
        processors:
          - resourcedetection/azure
          - resourcedetection/aks
          - resourcedetection/k8snode
          - resourcedetection/system
        exporters:
          - otlp/agg
          - debug
      logs/journald:
        exporters:
          - otlp/agg
          - debug
        processors:
          - resourcedetection/azure
          - resourcedetection/aks
          - resourcedetection/k8snode
          - resourcedetection/system
        receivers:
          - journald
      metrics:
        processors:
          - resourcedetection/azure
          - resourcedetection/aks
          - resourcedetection/k8snode
          - resourcedetection/system
        exporters:
          - otlp/agg
          - debug
      traces:
        exporters:
          - otlp/agg
          - debug

cluster-values.yaml

mode: deployment
# image:
#   # If you want to use the core image `otel/opentelemetry-collector`, you also need to change `command.name` value to `otelcol`.
#   repository: ghcr.io/segateway/containers/container-open-telemetry-collector-debian
#   pullPolicy: IfNotPresent
#   # Overrides the image tag whose default is the chart appVersion.
#   tag: "0.87.0-otelcol-contrib-journald-1.0.0"
# command:
# name: bin/otelcol-contrib
presets:
  kubernetesEvents:
    enabled: true
  clusterMetrics:
    enabled: true

resources:
  limits:
    cpu: 100m
    memory: 200M

tolerations:
  - key: "CriticalAddonsOnly"
    operator: "Exists"

useGOMEMLIMIT: true

config:
  exporters:
    otlp/agg:
      endpoint: "http://gateway-opentelemetry-collector:4317"
      tls:
        insecure: true

  processors:
    resourcedetection/azure:
      detectors:
        - azure
      override: false
    resourcedetection/aks:
      detectors:
        - aks
      override: false
  service:
    pipelines:
      logs:
        processors:
          - resourcedetection/azure
          - resourcedetection/aks
        exporters:
          - otlp/agg
          - debug
      metrics:
        processors:
          - resourcedetection/azure
          - resourcedetection/aks
        exporters:
          - otlp/agg
          - debug

gateway-values.yaml

mode: deployment
# image:
#   # If you want to use the core image `otel/opentelemetry-collector`, you also need to change `command.name` value to `otelcol`.
#   repository: ghcr.io/segateway/containers/container-open-telemetry-collector-debian
#   pullPolicy: IfNotPresent
#   # Overrides the image tag whose default is the chart appVersion.
#   tag: "0.87.0-otelcol-contrib-journald-1.0.0"
# command:
# name: bin/otelcol-contrib
replicaCount: 3

presets:
  kubernetesAttributes:
    enabled: true
    extractAllPodLabels: true

resources:
  limits:
    cpu: 100m
    memory: 200M

tolerations:
  - key: "CriticalAddonsOnly"
    operator: "Exists"

useGOMEMLIMIT: true

extraVolumes:

config:
  exporters:
    otlphttp/agg:
      endpoint: "https://logscale-ps-inputs.gcp.logsr.life/api/v1/ingest/otlp"
      headers:
        Authorization: Bearer f8c8bd9d-8458-430f-afba-xxx
  processors:
    resourcedetection/azure:
      detectors:
        - azure
      override: false
    resourcedetection/aks:
      detectors:
        - aks
      override: false
    k8sattributes:
      extract:
        labels:
          - from: pod
            key_regex: (.*)
            tag_name: k8s.labels.$$1
        metadata:
          - k8s.namespace.name
          - k8s.deployment.name
          - k8s.statefulset.name
          - k8s.daemonset.name
          - k8s.cronjob.name
          - k8s.job.name
          - k8s.node.name
          - k8s.pod.name
          - k8s.pod.uid
          - k8s.pod.start_time
          - k8s.container.name
          - container.image.name
          - container.image.tag
          - container.id
      filter:
        node_from_env_var: K8S_NODE_NAME
      passthrough: false
      pod_association:
        - sources:
            - from: resource_attribute
              name: k8s.pod.ip
        - sources:
            - from: resource_attribute
              name: k8s.pod.uid
        - sources:
            - from: connection
  service:
    pipelines:
      logs:
        processors:
          - resourcedetection/azure
          - resourcedetection/aks
        exporters:
          - otlphttp/agg
          - debug
      metrics:
        processors:
          - resourcedetection/azure
          - resourcedetection/aks
        exporters:
          - otlphttp/agg
          - debug
      traces:
        exporters:
          # - otlp
          - debug

otel-operator-values.yaml

replicaCount: 2
pdb:
  ## Enable/disable a Pod Disruption Budget creation
  ##
  create: false
  ## Minimum number/percentage of pods that should remain scheduled
  ##
  minAvailable: 1

tolerations:
  - key: "CriticalAddonsOnly"
    operator: "Exists"
manager:    
  image:
    tag: v0.87.0
collectorImage:
    repository: ghcr.io/segateway/containers/container-open-telemetry-collector-debian
    tag: 0.87.0-otelcol-contrib-base-1.0.0

parser.yaml

name: otel-logscale
tests:
- 2018-10-15T12:51:40+00:00 [INFO] This is an example log entry. id=123 fruit=banana
- 2018-10-15T12:52:42+01:30 [ERROR] Here is an error log entry. class=c.o.StringUtil
  fruit=pineapple
- 2018-10-15T12:53:12+01:00 [INFO] User logged in. user_id=1831923 protocol=http
fieldsToBeRemovedBeforeParsing: []
$schema: https://schemas.humio.com/parser/v0.2.0
script: |-
  | case {
    "scopeLog.logRecord.body"=*
      | case {
        "scopeLog.logRecord.body" = /^{.*}$/
          | parseJson("scopeLog.logRecord.body", prefix="scopeLog.logRecord.attributes.")
          | scopeLog.logRecord.attributes.body.format := "json"
          | drop([scopeLog.logRecord.body]);
        scopeLog.logRecord.body=/^(?<zheader>.+)(?<zdata>{.*})$/
          | parseJson("zdata", prefix="scopeLog.logRecord.attributes.")
          | case {
              "@error_msg" = "Could not parse json for field=zdata msg=Could not handle input. reason=Could not parse JSON"
                  | drop(fields=[zheader,zdata,@error_msg,@error_msg[0]]);
              *
                  | scopeLog.logRecord.attributes.msgHeader := rename(zheader)
                  | drop(fields=[zdata])
                  | scopeLog.logRecord.attributes.body.format := "header_json";
          };
        scopeLog.logRecord.body=/(?<zheader>.+?) ?(?<zdata>(?:"?[a-zA-Z0-9\-\_]+"?=(?:"[^"]*?"|[^ ]+) ?)+)$/
          | kvParse(field="zdata", as="scopeLog.logRecord.attributes")
          | case {
              zheader = *
                  | scopeLog.logRecord.attributes.msgHeader := rename(zheader)
                  | scopeLog.logRecord.attributes.body.format := "header_kv";
              * | scopeLog.logRecord.attributes.body.format := "kv";
          }
          | drop(fields=[zdata]);
        * | scopeLog.logRecord.attributes.body.format := "unknown"
      }

      | case {
        "resource.attributes.container.image.name" = "humio/humio-core"
          | @kind := rename(scopeLog.logRecord.attributes.kind)
          | class := rename(scopeLog.logRecord.attributes.class)
          | loglevel := rename(scopeLog.logRecord.attributes.loglevel)
          | case {
              "scopeLog.logRecord.attributes.message" = *
                      | @rawstring := rename(scopeLog.logRecord.attributes.message)
                      | drop([scopeLog.logRecord.attributes.message]);
                  *
          }
          | thread := rename(scopeLog.logRecord.attributes.thread)
          | @type := "humio"
          | @vhost := rename(scopeLog.logRecord.attributes.vhost)
          | kvParse();
        *
      };
    *
  }
tagFields:
- '@kind'
- '@vhost'
- '@type'