Humio Parser template

Kv-withfw.yaml

name: Kv-withfw
tests:
- '{"@timestamp":"2023-03-13T08:50:43.847508+13:00","message":" time=\"2023-03-12T19:50:43Z\"
  level=info msg=\"Note that the first check will be performed in 23 hours, 59 minutes,
  59 seconds\"","host":"melkor","HOST":"melkor","severity":"err","facility":"user","syslogtag":"watchtower/cf3b2dbd4c2f[1167751]:","name":"watchtower","pid":"1167751"}'
- '{"SOURCE":"s_network_udp","PROGRAM":"brilliant_02-1910","PRIORITY":"notice","MESSAGE":"ESP-MQT:
  sensors/tasmota/stat/brilliant_02/POWER = OFF","LEGACY_MSGHDR":"brilliant_02-1910
  ","HOST_FROM":"tasmota-1910","HOST":"tasmota-1910","FACILITY":"user","@timestamp":"2022-12-17T15:40:52+13:00"}'
- '{"SOURCE":"s_network_udp","PROGRAM":"backup-openmediavault/72419837d8","PRIORITY":"info","MESSAGE":"","LEGACY_MSGHDR":"backup-openmediavault/72419837d8
  ","HOST_FROM":"drogo","HOST":"drogo","FACILITY":"user","@timestamp":"2022-12-09T11:15:56+13:00"}'
- '{"@timestamp":"2022-12-09T19:50:07.533163+13:00","message":" {\"status\":1,\"request\":\"63bc5e5c-e70b-456e-b81c-73b0b3474f22\"}","host":"drogo","HOST":"drogo","severity":"info","facility":"user","syslogtag":"restic-backup/e232296ef327[2408999]:","name":"restic-backup","pid":"2408999"}'
- '{"SOURCE":"s_network_udp","PROGRAM":"kernel","PRIORITY":"warning","MESSAGE":"[WAN_LOCAL-default-D]IN=pppoe2
  OUT= MAC= SRC=46.71.51.195 DST=203.86.195.69 LEN=60 TOS=0x00 PREC=0x20 TTL=47 ID=8954
  DF PROTO=TCP SPT=40494 DPT=5555 WINDOW=65535 RES=0x00 SYN URGP=0 ","LEGACY_MSGHDR":"kernel:
  ","HOST_FROM":"USG","HOST":"USG","FACILITY":"kern","@timestamp":"2022-12-09T16:38:23+13:00"}'
- '{"@timestamp":"2022-12-09T14:35:00.981200+13:00","message":" {\"level\":\"info\",\"ts\":1670549700.980448,\"logger\":\"http.log.access\",\"msg\":\"handled
  request\",\"request\":{\"remote_ip\":\"192.168.10.50\",\"remote_port\":\"54163\",\"proto\":\"HTTP/2.0\",\"method\":\"GET\",\"host\":\"sonarr.drogo-internal.testing.nz\",\"uri\":\"/sonarr/feed/calendar/Sonarr.ics?apikey=20e93a5e68a344fb9df3dd47e5dcd02a\",\"headers\":{\"Accept-Encoding\":[\"gzip\"],\"Accept\":[\"text/calendar,
  application/calendar+json, application/calendar+xml\"],\"User-Agent\":[\"Nextcloud
  Webcal Crawler\"],\"X-Forwarded-For\":[\"172.24.0.1\"]},\"tls\":{\"resumed\":false,\"version\":772,\"cipher_suite\":4865,\"proto\":\"h2\",\"server_name\":\"sonarr.drogo-internal.testing.nz\"}},\"user_id\":\"\",\"duration\":0.0000072,\"size\":0,\"status\":0,\"resp_headers\":{\"Server\":[\"Caddy\"]}}","host":"drogo","HOST":"drogo","severity":"err","facility":"user","syslogtag":"caddy/55573fe6102c[2408999]:","name":"caddy","pid":"2408999"}'
- '{"@timestamp":"2022-12-08T09:22:43.798846+13:00","message":" {\"level\":\"info\",\"ts\":1670444563.798683,\"logger\":\"docker-proxy\",\"msg\":\"New
  Config JSON\",\"json\":\"{\\\"logging\\\":{\\\"logs\\\":{\\\"default\\\":{\\\"exclude\\\":[\\\"http.log.access.log1\\\",\\\"http.log.access.log0\\\"]},\\\"log0\\\":{\\\"writer\\\":{\\\"filename\\\":\\\"/var/log/caddy/access-internal.log\\\",\\\"output\\\":\\\"file\\\"},\\\"include\\\":[\\\"http.log.access.log0\\\"]},\\\"log1\\\":{\\\"writer\\\":{\\\"filename\\\":\\\"/var/log/caddy/access.log\\\",\\\"output\\\":\\\"file\\\"},\\\"include\\\":[\\\"http.log.access.log1\\\"]}}},\\\"apps\\\":{\\\"http\\\":{\\\"servers\\\":{\\\"srv0\\\":{\\\"listen\\\":[\\\":443\\\"],\\\"routes\\\":[{\\\"match\\\":[{\\\"host\\\":[\\\"frodo-internal.testing.nz\\\"]}],\\\"terminal\\\":true},{\\\"match\\\":[{\\\"host\\\":[\\\"internal.testing.nz\\\"]}],\\\"terminal\\\":true},{\\\"match\\\":[{\\\"host\\\":[\\\"*.frodo-internal.testing.nz\\\"]}],\\\"terminal\\\":true},{\\\"match\\\":[{\\\"host\\\":[\\\"*.internal.testing.nz\\\"]}],\\\"handle\\\":[{\\\"handler\\\":\\\"subroute\\\",\\\"routes\\\":[{\\\"handle\\\":[{\\\"handler\\\":\\\"reverse_proxy\\\",\\\"upstreams\\\":[{\\\"dial\\\":\\\"172.24.0.2:80\\\"}]}],\\\"match\\\":[{\\\"host\\\":[\\\"bitwarden.internal.testing.nz\\\"]}]}]}],\\\"terminal\\\":true}],\\\"logs\\\":{\\\"logger_names\\\":{\\\"*.internal.testing.nz\\\":\\\"log0\\\",\\\"internal.testing.nz\\\":\\\"log1\\\"}}}}},\\\"tls\\\":{\\\"automation\\\":{\\\"policies\\\":[{\\\"subjects\\\":[\\\"frodo-internal.testing.nz\\\",\\\"internal.testing.nz\\\",\\\"*.frodo-internal.testing.nz\\\",\\\"*.internal.testing.nz\\\"],\\\"issuers\\\":[{\\\"challenges\\\":{\\\"dns\\\":{\\\"provider\\\":{\\\"api_token\\\":\\\"8HOtzaBBY7rKHDlClkQ7INmALS03f0QCZF5sdMlV\\\",\\\"name\\\":\\\"cloudflare\\\"}}},\\\"email\\\":\\\"[email protected]\\\",\\\"module\\\":\\\"acme\\\"},{\\\"challenges\\\":{\\\"dns\\\":{\\\"provider\\\":{\\\"api_token\\\":\\\"8HOtzaBBY7rKHDlClkQ7INmALS03f0QCZF5sdMlV\\\",\\\"name\\\":\\\"cloudflare\\\"}}},\\\"email\\\":\\\"[email protected]\\\",\\\"module\\\":\\\"zerossl\\\"}]}]}}}}\"}","host":"frodo","severity":"err","facility":"user","syslogtag":"caddy/bfda448ba56e[955288]:","name":"caddy","pid":"955288"}'
- '{"@timestamp":"2022-12-09T06:48:09.750716+13:00","message":" 2022/12/09 06:48:09
  #011/backups/influxdb/20221208T174559Z.s3395.tar.gz","host":"drogo","HOST":"drogo","severity":"info","facility":"user","syslogtag":"restic-backup/e232296ef327[2408999]:","name":"restic-backup","pid":"2408999"}'
- <34>1 2003-10-11T22:14:15.003Z mymachine.example.com su - ID47 - BOM'su root' failed
  for lonvick on /dev/pts/8
- '{"Timestamp":"2019-01-18T13:06:53.3642110+01:00","Level":"Information","MessageTemplate":"Number
  of user sessions {@UserSessions}","RenderedMessage":"Number of user sessions 103","Properties":{"UserSessions":103,"ProcessId":"8009"}}'
- |
  {"SOURCE":"s_network_udp","PROGRAM":"dhcpd","PRIORITY":"debug","MESSAGE":"execute_statement argv[3] = 192.168.100.39","LEGACY_MSGHDR":"dhcpd: ","HOST_FROM":"USG","HOST":"USG","FACILITY":"daemon","@timestamp":"2022-12-09T07:36:02+13:00"}
- '{"SOURCE":"s_network_udp","PROGRAM":"mcad","PRIORITY":"info","MESSAGE":"mcad[4777]:
  perl_wrapper.perl_request_response(): exit with ret 32","LEGACY_MSGHDR":"mcad: ","HOST_FROM":"USG","HOST":"USG","FACILITY":"user","@timestamp":"2022-12-09T16:21:01+13:00"}'
fieldsToBeRemovedBeforeParsing: []
$schema: https://schemas.humio.com/parser/v0.2.0
script: |+
  //parse caddy line
  kvParse(@rawstring) | findTimestamp(addErrors=false) |
  //Find some json in teh raw string and parse it
  case {
      /\{.+\}/
      | parseJson(field=@rawstring) |

      case {
          //Caddy Jason line
          name=caddy
          |     findTimestamp(addErrors=false) |  parseJson(field=message) | message :=MESSAGE | parserule:="caddy";

          //USG Firewall line
          /.+PROGRAM\":\"kernel.+MESSAGE.+\[(?<fwrule>.+)\]IN=(?<IN>\S+)\s.+/
          |kvParse(MESSAGE) | host:= HOST | message :=MESSAGE | findTimestamp(field=@timestamp) | parserule:="usgfw";

          //USG line
          /.+MESSAGE.+/
          //rfeset the host variable
          |host:= HOST | findTimestamp(field=@timestamp) | message :=MESSAGE | parserule:="usg";

          //Other docker
          // / time\=.+level\=(?<level>.+).msg\=(?<message>.+).host\=/
          / time\=.+level\=(?<level>.+) msg\=\\"(?<message>.+)\\/ | findTimestamp(addErrors=false)  | parserule:="docker";

          //Anything else
          * |
          findTimestamp(addErrors=false,field=@timestamp) | parserule:="jsoncatchall";
      };
      //Anything else
      *
      |findTimestamp(addErrors=false) | findTimestamp(addErrors=false,field=@timestamp) | parserule:="catchall";
  }

tagFields: []