Mtail metrics exporting utilities

nginx-combined.mtail

# This file is available under the Apache license.

# TARGET_OS=linux
# TARGET_ARCH=amd64
# mtail_latest="$(curl -qsSL https://api.github.com/repos/google/mtail/releases | jq '.[].html_url' | grep '[0-9].[0-9].[0-9]' | grep -v '\(dev\|next\|alpha\|beta\|rc[0-9]\+\)' | awk -F'/v' '{print $NF}' | cut -d'"' -f1 | sort -V -r | head -n 1)"
# curl -qsSL -o "mtail_${mtail_latest}_${TARGET_OS}_${TARGET_ARCH}.tar.gz" "https://github.com/google/mtail/releases/download/v${mtail_latest}/mtail_${mtail_latest}_${TARGET_OS}_${TARGET_ARCH}.tar.gz"
# [[ $? -eq 0 ]] || exit 10
# curl -qsSL -o- "https://github.com/google/mtail/releases/download/v${mtail_latest}/checksums.txt" | grep "mtail_${mtail_latest}_${TARGET_OS}_${TARGET_ARCH}.tar.gz" | tee "mtail_${mtail_latest}_${TARGET_OS}_${TARGET_ARCH}.tar.gz.sha256" 1>/dev/null
# grep "$(sha256sum "mtail_${mtail_latest}_${TARGET_OS}_${TARGET_ARCH}.tar.gz" )" "mtail_${mtail_latest}_${TARGET_OS}_${TARGET_ARCH}.tar.gz.sha256" && tar xvzf "mtail_${mtail_latest}_${TARGET_OS}_${TARGET_ARCH}.tar.gz" && sudo mv mtail /usr/local/bin/mtail
#
# Nginx Combined Log (default) based on "NCSA extended/combined" log format
# log_format combined '$remote_addr - $remote_user [$time_local] '
#                     '"$request" $status $body_bytes_sent '
#                     '"$http_referer" "$http_user_agent"';
counter nginx_http_requests_total by request_method, http_version, request_status #, uri
counter nginx_http_bytes_total by request_method, http_version, request_status

/^/ +
/(?P<hostname>[0-9A-Za-z\.:-]+)\s+/ + # %h
/-\s+/ + # /(?P<remote_logname>[0-9A-Za-z-]+) / + # %l
/(?P<remote_username>[0-9A-Za-z-]+)\s+/ + # %u
/\[(?P<timestamp>\d{2}\/\w{3}\/\d{4}:\d{2}:\d{2}:\d{2} (\+|-)\d{4})\]\s+/ + # %t
/"((?P<request_method>HEAD|GET|POST|PUT|PATCH|CONNECT|DELETE|OPTIONS|TRACE)|(?:\S+))\s+(?P<uri>\/[^\?\n\r\t\s\v]*|\/)\S*\s+((?P<http_version>HTTP\/[0-9\.]+)|(?:\S+))"\s+/ + # \"%r\"
/(?P<request_status>\d{3})\s+/ + # %>s
/((?P<response_size>\d+)|-)\s+/ + # %b
/"(?P<referer>\S+)"\s+/ + # \"%{Referer}i\"
/"(?P<user_agent>[[:print:]]+)"/ + # \"%{User-agent}i\"
/$/ {
  strptime($timestamp, "02/Jan/2006:15:04:05 -0700") # for tests

  # uri = subs(/\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/, "/:uuid:", $uri)
  # uri = subst(/\/d+/, "/:num:", $uri)
  nginx_http_requests_total[$request_method][$http_version][$request_status]++

  $response_size > 0 {
      nginx_http_bytes_total[$request_method][$http_version][$request_status] += $response_size
  }
}

nginx-upstreaminfo.mtail

# This file is available under the Apache license.

# TARGET_OS=linux
# TARGET_ARCH=amd64
# mtail_latest="$(curl -qsSL https://api.github.com/repos/google/mtail/releases | jq '.[].html_url' | grep '[0-9].[0-9].[0-9]' | grep -v '\(dev\|next\|alpha\|beta\|rc[0-9]\+\)' | awk -F'/v' '{print $NF}' | cut -d'"' -f1 | sort -V -r | head -n 1)"
# curl -qsSL -o "mtail_${mtail_latest}_${TARGET_OS}_${TARGET_ARCH}.tar.gz" "https://github.com/google/mtail/releases/download/v${mtail_latest}/mtail_${mtail_latest}_${TARGET_OS}_${TARGET_ARCH}.tar.gz"
# [[ $? -eq 0 ]] || exit 10
# curl -qsSL -o- "https://github.com/google/mtail/releases/download/v${mtail_latest}/checksums.txt" | grep "mtail_${mtail_latest}_${TARGET_OS}_${TARGET_ARCH}.tar.gz" | tee "mtail_${mtail_latest}_${TARGET_OS}_${TARGET_ARCH}.tar.gz.sha256" 1>/dev/null
# grep "$(sha256sum "mtail_${mtail_latest}_${TARGET_OS}_${TARGET_ARCH}.tar.gz" )" "mtail_${mtail_latest}_${TARGET_OS}_${TARGET_ARCH}.tar.gz.sha256" && tar xvzf "mtail_${mtail_latest}_${TARGET_OS}_${TARGET_ARCH}.tar.gz" && sudo mv mtail /usr/local/bin/mtail
#
# Nginx upstreaminfo Log (K8s, Alibaba, OVH-style field names) based on
# "NCSA extended/combined" with upstream timing and size fields.
# 
# Future-proof variant:
# - Parses the known upstreaminfo fields.
# - Allows extra fields to be appended at the end of the log line.
# - Captures those trailing fields as extra_fields, but does not use them in metrics.
# 
# ingress-nginx default upstreaminfo:
# log_format upstreaminfo '$remote_addr - $remote_user [$time_local] "$request" '
#                         '$status $body_bytes_sent "$http_referer" "$http_user_agent" '
#                         '$request_length $request_time [$proxy_upstream_name] '
#                         '[$proxy_alternative_upstream_name] $upstream_addr '
#                         '$upstream_response_length $upstream_response_time '
#                         '$upstream_status $req_id';
#
# Older/compact variant also supported by the same parser:
#                         '$request_length $request_time [-] $upstream_addr '
#                         '$upstream_response_length $upstream_response_time '
#                         '$upstream_status';

counter http_requests_total by method, protocol, status #, uri
counter http_bytes_sent_total by method, protocol, status
counter http_bytes_recv_total by method, protocol, status
histogram http_request_duration_seconds by method, protocol, status buckets 0.001, 0.002, 0.003, 0.005, 0.007, 0.01, 0.015, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10
histogram http_upstream_response_duration_seconds by method, protocol, status buckets 0.001, 0.002, 0.003, 0.005, 0.007, 0.01, 0.015, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10
histogram http_response_bytes by method, protocol, status buckets 10,1000,100000,1e+06,5e+06,5e+07,2e+08,1e+09
histogram http_upstream_response_bytes by method, protocol, status buckets 10,1000,100000,1e+06,5e+06,5e+07,2e+08,1e+09
histogram http_request_bytes by method, protocol, status buckets 10,250,2000,10000,100000,1e+06,1e+07,1e+08

# OVH Cloud field naming convention reference:
# https://github.com/ovh/docs/blob/develop/pages/manage_and_operate/observability/logs_data_platform/ingestion_apache/guide.en-ca.md
# Kubernetes ingress-nginx upstreaminfo fields reference:
# https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/log-format/
# Alibaba Cloud WAF field naming reference:
# https://www.alibabacloud.com/help/en/waf/web-application-firewall-3-0/user-guide/fields-in-logs

/^/ +
/(?P<host>[0-9A-Za-z\.:-]+)\s+/ + # remote_addr / apache %h Remote host / Alibaba src_ip-like value
/-\s+/ + # remote_logname placeholder / apache %l, nginx combined '-' blank unsupported equivalent
/(?P<user>[0-9A-Za-z._@-]+|-)\s+/ + # remote_user / apache %u Remote user
/\[(?P<time>\d{2}\/\w{3}\/\d{4}:\d{2}:\d{2}:\d{2} (\+|-)\d{4})\]\s+/ + # time_local / apache %t Time the request was received
/"(?P<method>HEAD|GET|POST|PUT|PATCH|CONNECT|DELETE|OPTIONS|TRACE|[A-Z]+)\s+(?P<uri>\S+)\s+(?P<protocol>HTTP\/[0-9\.]+|\S+)"\s+/ + # request: method URI protocol apache \"%r\", first line of request => apache "%m %U%q %H" Request method + Request URI + Requested Protocol ("HTTP/1.0" "HTTP/1.1")
/(?P<status>\d{3})\s+/ + # status / apache %>s HTTP Status code / Alibaba status
/(?P<size_int>\d+)\s+/ + # body_bytes_sent / apache %b  Size of response in bytes, excluding HTTP headers. / Alibaba body_bytes_sent
/"(?P<referer>[^"]*)"\s+/ + # http_referer / apache \"%{Referer}i\" Referer header / Alibaba http_referer
/"(?P<user_agent>[^"]*)"\s+/ + # http_user_agent / apache \"%{User-agent}i\" User-Agent header / Alibaba http_user_agent
/(?P<reqsize_int>\d+)\s+/ + # request_length / apache %I  bytes received, including request and headers / Alibaba request_length
/(?P<reqtime_num>\d+(\.\d+)?)\s+/ + # request_time / apache %T, time taken to serve the request, in seconds
/\[(?P<proxy_upstream_name>[^\]]*)\]\s+/ + # proxy_upstream_name, or '-' in compact format
/(?:\[(?P<proxy_alternative_upstream_name>[^\]]*)\]\s+)?/ + # optional proxy_alternative_upstream_name
/(?P<upstream_addr>-|\S+(?:,\s*\S+)*)\s+/ + # upstream_addr / Alibaba upstream_addr
/(?:(?P<upstream_response_size_int>\d+)(?:,\s*\d+)*|-)\s+/ + # upstream_response_length; first value captured
/(?:(?P<upstream_response_time_num>\d+(\.\d+)?)(?:,\s*\d+(\.\d+)?)*|-)\s+/ + # upstream_response_time; first value captured
/(?P<upstream_status>-|\d{3}(?:,\s*\d{3})*)/ + # upstream_status / Alibaba upstream_status
/(?:\s+(?P<request_id>\S+))?/ + # req_id / request_traceid-like value, present in ingress-nginx
/(?:\s+(?P<extra_fields>[[:print:]]+))?/ + # future appended fields; intentionally not parsed yet
/$/ {
  strptime($time, "02/Jan/2006:15:04:05 -0700") # for tests

  # Optional URI normalization if you decide to add uri as a metric label.
  # uri = subs(/\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/, "/:uuid:", $uri)
  # uri = subst(/\/\d+/, "/:num:", $uri)

  http_requests_total[$method][$protocol][$status]++

  $size_int > 0 {
      http_bytes_sent_total[$method][$protocol][$status] += $size_int
  }

  $reqsize_int > 0 {
      http_bytes_recv_total[$method][$protocol][$status] += $reqsize_int
  }

  http_request_duration_seconds[$method][$protocol][$status] = $reqtime_num
  http_response_bytes[$method][$protocol][$status] = $size_int
  http_request_bytes[$method][$protocol][$status] = $reqsize_int

  $upstream_response_time_num > 0 {
      http_upstream_response_duration_seconds[$method][$protocol][$status] = $upstream_response_time_num
  }

  $upstream_response_size_int > 0 {
      http_upstream_response_bytes[$method][$protocol][$status] = $upstream_response_size_int
  }
}