Last active
July 1, 2025 22:39
-
-
Save drconopoima/c9c41c96591039752395757300d2516f to your computer and use it in GitHub Desktop.
Mtail metrics exporting utilities
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This file is available under the Apache license. | |
| # TARGET_OS=linux | |
| # TARGET_ARCH=amd64 | |
| # mtail_latest="$(curl -qfsSL https://api.github.com/repos/google/mtail/releases | jq '.[].html_url' | grep '[0-9].[0-9].[0-9]' | grep -v '\(dev\|next\|alpha\|beta\|rc[0-9]\+\)' | awk -F'/v' '{print $NF}' | cut -d'"' -f1 | sort -V -r | head -n 1)" | |
| # curl -qfsSL -o "mtail_${mtail_latest}_${TARGET_OS}_${TARGET_ARCH}.tar.gz" "https://github.com/google/mtail/releases/download/v${mtail_latest}/mtail_${mtail_latest}_${TARGET_OS}_${TARGET_ARCH}.tar.gz" | |
| # [[ $? -eq 0 ]] || exit 10 | |
| # curl -qfsSL -o- "https://github.com/google/mtail/releases/download/v${mtail_latest}/checksums.txt" | grep "mtail_${mtail_latest}_${TARGET_OS}_${TARGET_ARCH}.tar.gz" | tee "mtail_${mtail_latest}_${TARGET_OS}_${TARGET_ARCH}.tar.gz.sha256" 1>/dev/null | |
| # grep "$(sha256sum "mtail_${mtail_latest}_${TARGET_OS}_${TARGET_ARCH}.tar.gz" )" "mtail_${mtail_latest}_${TARGET_OS}_${TARGET_ARCH}.tar.gz.sha256" && tar xvzf "mtail_${mtail_latest}_${TARGET_OS}_${TARGET_ARCH}.tar.gz" && sudo mv mtail /usr/local/bin/mtail | |
| # | |
| # Nginx Combined Log (default) based on "NCSA extended/combined" log format | |
| # log_format combined '$remote_addr - $remote_user [$time_local] ' | |
| # '"$request" $status $body_bytes_sent ' | |
| # '"$http_referer" "$http_user_agent"'; | |
| counter nginx_http_requests_total by request_method, http_version, request_status #, uri | |
| counter nginx_http_bytes_total by request_method, http_version, request_status | |
| /^/ + | |
| /(?P<hostname>[0-9A-Za-z\.:-]+)\s+/ + # %h | |
| /-\s+/ + # /(?P<remote_logname>[0-9A-Za-z-]+) / + # %l | |
| /(?P<remote_username>[0-9A-Za-z-]+)\s+/ + # %u | |
| /\[(?P<timestamp>\d{2}\/\w{3}\/\d{4}:\d{2}:\d{2}:\d{2} (\+|-)\d{4})\]\s+/ + # %t | |
| /"((?P<request_method>HEAD|GET|POST|PUT|PATCH|CONNECT|DELETE|OPTIONS|TRACE)|(?:\S+))\s+(?P<uri>\/[^\?\n\r\t\s\v]*|\/)\S*\s+((?P<http_version>HTTP\/[0-9\.]+)|(?:\S+))"\s+/ + # \"%r\" | |
| /(?P<request_status>\d{3})\s+/ + # %>s | |
| /((?P<response_size>\d+)|-)\s+/ + # %b | |
| /"(?P<referer>\S+)"\s+/ + # \"%{Referer}i\" | |
| /"(?P<user_agent>[[:print:]]+)"/ + # \"%{User-agent}i\" | |
| /$/ { | |
| strptime($timestamp, "02/Jan/2006:15:04:05 -0700") # for tests | |
| # uri = subs(/\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/, "/:uuid:", $uri) | |
| # uri = subst(/\/d+/, "/:num:", $uri) | |
| nginx_http_requests_total[$request_method][$http_version][$request_status]++ | |
| $response_size > 0 { | |
| nginx_http_bytes_total[$request_method][$http_version][$request_status] += $response_size | |
| } | |
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This file is available under the Apache license. | |
| # TARGET_OS=linux | |
| # TARGET_ARCH=amd64 | |
| # mtail_latest="$(curl -qfsSL https://api.github.com/repos/google/mtail/releases | jq '.[].html_url' | grep '[0-9].[0-9].[0-9]' | grep -v '\(dev\|next\|alpha\|beta\|rc[0-9]\+\)' | awk -F'/v' '{print $NF}' | cut -d'"' -f1 | sort -V -r | head -n 1)" | |
| # curl -qfsSL -o "mtail_${mtail_latest}_${TARGET_OS}_${TARGET_ARCH}.tar.gz" "https://github.com/google/mtail/releases/download/v${mtail_latest}/mtail_${mtail_latest}_${TARGET_OS}_${TARGET_ARCH}.tar.gz" | |
| # [[ $? -eq 0 ]] || exit 10 | |
| # curl -qfsSL -o- "https://github.com/google/mtail/releases/download/v${mtail_latest}/checksums.txt" | grep "mtail_${mtail_latest}_${TARGET_OS}_${TARGET_ARCH}.tar.gz" | tee "mtail_${mtail_latest}_${TARGET_OS}_${TARGET_ARCH}.tar.gz.sha256" 1>/dev/null | |
| # grep "$(sha256sum "mtail_${mtail_latest}_${TARGET_OS}_${TARGET_ARCH}.tar.gz" )" "mtail_${mtail_latest}_${TARGET_OS}_${TARGET_ARCH}.tar.gz.sha256" && tar xvzf "mtail_${mtail_latest}_${TARGET_OS}_${TARGET_ARCH}.tar.gz" && sudo mv mtail /usr/local/bin/mtail | |
| # | |
| # Nginx upstreaminfo Log (K8s, Alibaba) based on "NCSA extended/combined" log format | |
| # log_format upstreaminfo '$remote_addr - $remote_user [$time_local] ' | |
| # '"$request" $status $body_bytes_sent ' | |
| # '"$http_referer" "$http_user_agent" $request_length ' | |
| # '$request_time [-] $upstream_addr ' | |
| # '$upstream_response_length $upstream_response_time ' | |
| # '$upstream_status'; | |
| counter http_requests_total by method, protocol, status #, uri | |
| counter http_bytes_sent_total by method, protocol, status | |
| counter http_bytes_recv_total by method, protocol, status | |
| histogram http_request_duration_seconds by method, protocol, status buckets 0.001, 0.002, 0.003, 0.005, 0.007, 0.01, 0.015, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10 | |
| histogram http_response_bytes by method, protocol, status buckets 10,1000,100000,1e+06,5e+06,5e+07,2e+08,1e+09 | |
| histogram http_request_bytes by method, protocol, status buckets 10,250,2000,10000,100000,1e+06,1e+07,1e+08 | |
| # OVH Cloud Field Naming conventions https://github.com/ovh/docs/blob/develop/pages/manage_and_operate/observability/logs_data_platform/ingestion_apache/guide.en-ca.md | |
| /^/ + | |
| /(?P<host>[0-9A-Za-z\.:-]+)\s+/ + # apache %h Remote host | |
| /-\s+/ + # /(?P<remote_logname>[0-9A-Za-z-]+) / + # apache %l Remote logname, nginx combined '-' blank unsupported equivalent | |
| /(?P<user>[0-9A-Za-z-]+)\s+/ + # apache %u Remote user | |
| /\[(?P<time>\d{2}\/\w{3}\/\d{4}:\d{2}:\d{2}:\d{2} (\+|-)\d{4})\]\s+/ + # apache %t Time the request was received | |
| /"((?P<method>HEAD|GET|POST|PUT|PATCH|CONNECT|DELETE|OPTIONS|TRACE)|(?:\S+))\s+(?P<uri>\/[^\?\n\r\t\s\v]*|\/)\S*\s+((?P<protocol>HTTP\/[0-9\.]+)|(?:\S+))"\s+/ + # apache \"%r\", first line of request => apache "%m %U%q %H" Request method + Request URI + Requested Protocol ("HTTP/1.0" "HTTP/1.1") | |
| /(?P<status>\d{3})\s+/ + # apache %>s HTTP Status code | |
| /((?P<size_int>\d+)|-)\s+/ + # apache %b Size of response in bytes, excluding HTTP headers. | |
| /"(?P<referer>\S+)"\s+/ + # apache \"%{Referer}i\", Referer header | |
| /"(?P<ua>[[:print:]]+)"/ + # apache \"%{User-agent}i\", User-Agent header | |
| /(?:\s+)?((?P<reqsize_int>\d+)|(?:\S+))?(?:\s+)?/ + # apache %I, bytes received, including request and headers | |
| /((?P<reqtime_num>\d+\.\d+)|(?:\S+))?(?:\s+)?/ + # apache %T, time taken to serve the request, in seconds | |
| /[[:print:]]+/ + | |
| /$/ { | |
| strptime($time, "02/Jan/2006:15:04:05 -0700") # for tests | |
| http_requests_total[$method][$protocol][$status][$uri]++ | |
| $size_int > 0 { | |
| http_bytes_sent_total[$method][$protocol][$status] += $size_int | |
| } | |
| $reqsize_int > 0 { | |
| http_bytes_recv_total[$method][$protocol][$status] += $reqsize_int | |
| } | |
| # uri = subs(/\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/, "/:uuid:", $uri) | |
| # uri = subst(/\/d+/, "/:num:", $uri) | |
| http_request_duration_seconds[$method][$protocol][$status][$uri] = $reqtime_num / 1.0 | |
| http_response_bytes[$method][$protocol][$status] = $size_int | |
| http_request_bytes[$method][$protocol][$status] = $reqsize_int | |
| } | |
| # apptime_num $upstream_response_time # Response time from the upstream server |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment