Use vector.dev to process Linux netconsole messages

vector-netconsole.yaml

---
# See https://www.kernel.org/doc/Documentation/networking/netconsole.txt
# Use e.g. `insmod netconsole.ko netconsole=+@/,[email protected]/` on the VM

# $ vector --version
# vector 0.34.2 (aarch64-unknown-linux-musl d685a16 2024-01-02 14:59:54.890517436)

sources:
  in:
    #type: "syslog" # does not work with netconsole
    type: "socket"
    address: "172.16.0.1:6666"
    mode: "udp"
    decoding:
      codec: "bytes"

transforms:
  netconsole_parse:
    inputs:
      - "in"
    type: "remap"
    # <level>,<sequnum>,<timestamp>,<contflag>;<message text>
    source: |
      parts = split(string!(.message), ";", limit: 2)
      details, err = split(parts[0], ",", limit: 4)
      contflag = string!(details[3])
      message = strip_whitespace!(parts[1])
      parsed_level = parse_int!(details[0], base: 10)

      # If we have comma inside contflag, then we have more header options
      if contains(contflag, ",") {
        # Split contflag and fix up value
        cont = split(contflag, ",")
        contflag = cont[0]

        # Process additional flags
        .addn_flags = {}
        for_each(slice!(cont, start: 1)) -> |_idx, value| {
          kv = split(value, "=", limit: 2)
          k = string(kv[0]) ?? ""
          v = kv[1]

          .addn_flags = set!(.addn_flags, [k], v)
        }
      }

      .cid = community_id!(source_ip: .host, source_port: .port, destination_ip: "172.16.0.1", destination_port: 6666, protocol: 17)
      .level = to_syslog_level(parsed_level) ?? parsed_level
      .sequnum = parse_int!(details[1], base: 10)
      .usec_since_boot = parse_int!(details[2], base: 10)
      .contflag = contflag

      #.timestamp = parse_int!(slice!(details[2], start: 0, end: -6), base: 10)
      #.timestamp_us_part = parse_int!(slice!(details[2], start: -6))

      .message = message

  netconsole_route:
    type: "route"
    inputs:
      - "netconsole_parse"
    route:
      "fragged":
        type: "vrl"
        source: |
          .addn_flags.ncfrag != null

  netconsole_reduce:
    type: "reduce"
    inputs:
      - "netconsole_route.fragged"
    group_by:
      - "cid"
      - "sequnum"
    merge_strategies:
      "level": "discard"
      "message": "concat_raw"
      "port": "discard"
      "sequnum": "discard"
      "usec_since_boot": "discard"

    expire_after_ms: 250
    flush_period_ms: 500

    starts_when:
      type: "vrl"
      source: |
        is_frag = false

        ncf = .addn_flags.ncfrag
        if ncf != null {
          parts = split(string!(ncf), "/", limit: 2)
          is_frag = parts[0] == "0"
        }

        is_frag

sinks:
  out:
    inputs:
      - "netconsole_reduce"
      - "netconsole_route._unmatched"
    type: "console"
    encoding:
      codec: "json"

Example: https://gist.github.com/mikroskeem/9d5c6bb8c7c8dd2e6dc88a635497c576 (echo c > /proc/sysrq-trigger)