Skip to content

Instantly share code, notes, and snippets.

@krzysztofantczak

krzysztofantczak/dashboard.json

Created May 28, 2024 06:45
Show Gist options
  • Save krzysztofantczak/f4ba521da1d321f8dfa32c3217fb6514 to your computer and use it in GitHub Desktop.
Save krzysztofantczak/f4ba521da1d321f8dfa32c3217fb6514 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
dashboard.json
{
"__inputs": [
{
"name": "DS_VICTORIA",
"label": "victoria",
"description": "",
"type": "datasource",
"pluginId": "prometheus",
"pluginName": "Prometheus"
}
],
"__elements": {},
"__requires": [
{
"type": "grafana",
"id": "grafana",
"name": "Grafana",
"version": "11.0.0"
},
{
"type": "panel",
"id": "piechart",
"name": "Pie chart",
"version": ""
},
{
"type": "datasource",
"id": "prometheus",
"name": "Prometheus",
"version": "1.0.0"
},
{
"type": "panel",
"id": "timeseries",
"name": "Time series",
"version": ""
}
],
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": null,
"links": [],
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_VICTORIA}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
}
},
"mappings": [],
"unit": "percent"
},
"overrides": []
},
"gridPos": {
"h": 10,
"w": 8,
"x": 0,
"y": 0
},
"id": 5,
"options": {
"legend": {
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"pieType": "pie",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"tooltip": {
"maxHeight": 600,
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_VICTORIA}"
},
"disableTextWrap": false,
"editorMode": "code",
"expr": "exp_process_percent_cpu > 1",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": false,
"legendFormat": "{{command}}",
"range": true,
"refId": "A",
"useBackend": false
}
],
"title": "Process CPU %",
"transparent": true,
"type": "piechart"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_VICTORIA}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": 900000,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "percent"
},
"overrides": []
},
"gridPos": {
"h": 10,
"w": 16,
"x": 8,
"y": 0
},
"id": 3,
"options": {
"legend": {
"calcs": [
"min",
"max",
"mean",
"last"
],
"displayMode": "table",
"placement": "right",
"showLegend": true,
"sortBy": "Last",
"sortDesc": true
},
"tooltip": {
"maxHeight": 600,
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_VICTORIA}"
},
"disableTextWrap": false,
"editorMode": "code",
"expr": "exp_process_percent_cpu > 1",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": false,
"legendFormat": "{{command}}",
"range": true,
"refId": "A",
"useBackend": false
}
],
"title": "Process CPU %",
"transparent": true,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_VICTORIA}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
}
},
"mappings": []
},
"overrides": []
},
"gridPos": {
"h": 10,
"w": 8,
"x": 0,
"y": 10
},
"id": 4,
"options": {
"legend": {
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"pieType": "pie",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"tooltip": {
"maxHeight": 600,
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_VICTORIA}"
},
"disableTextWrap": false,
"editorMode": "code",
"expr": "exp_process_percent_mem > 1",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": false,
"legendFormat": "{{command}}",
"range": true,
"refId": "A",
"useBackend": false
}
],
"title": "Process MEM %",
"transparent": true,
"type": "piechart"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_VICTORIA}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": 900000,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "percent"
},
"overrides": []
},
"gridPos": {
"h": 10,
"w": 16,
"x": 8,
"y": 10
},
"id": 6,
"options": {
"legend": {
"calcs": [
"min",
"max",
"mean",
"last"
],
"displayMode": "table",
"placement": "right",
"showLegend": true,
"sortBy": "Last",
"sortDesc": true
},
"tooltip": {
"maxHeight": 600,
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_VICTORIA}"
},
"disableTextWrap": false,
"editorMode": "code",
"expr": "exp_process_percent_mem > 1",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": false,
"legendFormat": "{{command}}",
"range": true,
"refId": "A",
"useBackend": false
}
],
"title": "Process MEM %",
"transparent": true,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_VICTORIA}"
},
"description": "Virtual Size: The virtual memory usage of entire task in bytes",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineStyle": {
"fill": "solid"
},
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": 900000,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"fieldMinMax": false,
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "decbytes"
},
"overrides": []
},
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 20
},
"id": 1,
"options": {
"legend": {
"calcs": [
"min",
"max",
"mean",
"last"
],
"displayMode": "table",
"placement": "right",
"showLegend": true,
"sortBy": "Last",
"sortDesc": true
},
"tooltip": {
"maxHeight": 600,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "11.0.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_VICTORIA}"
},
"disableTextWrap": false,
"editorMode": "code",
"expr": "exp_process_vsz > 1000000000",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": false,
"legendFormat": "{{command}}",
"range": true,
"refId": "A",
"useBackend": false
}
],
"title": "Process VSZ",
"transparent": true,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_VICTORIA}"
},
"description": "Resident Set Size: The non-swapped physical memory used by the task in bytes.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineStyle": {
"fill": "solid"
},
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"fieldMinMax": false,
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "decbytes"
},
"overrides": []
},
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 30
},
"id": 2,
"options": {
"legend": {
"calcs": [
"min",
"max",
"mean",
"last"
],
"displayMode": "table",
"placement": "right",
"showLegend": true,
"sortBy": "Last",
"sortDesc": true
},
"tooltip": {
"maxHeight": 600,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "11.0.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_VICTORIA}"
},
"disableTextWrap": false,
"editorMode": "code",
"expr": "exp_process_rss > 1000000000",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": false,
"legendFormat": "{{command}}",
"range": true,
"refId": "A",
"useBackend": false
}
],
"title": "Process RSS",
"transparent": true,
"type": "timeseries"
}
],
"refresh": "5s",
"schemaVersion": 39,
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-30m",
"to": "now"
},
"timeRangeUpdatedDuringEditOrView": false,
"timepicker": {},
"timezone": "browser",
"title": "Process Memory",
"uid": "edmzvjqv59wxse",
"version": 31,
"weekStart": ""
}
Raw
process.py
from datetime import datetime
import psutil
import time
import os
import re
def check_exclusions(command):
exclusions = [
"kworker", "agetty", "systemd", "sssd", "sshd", "docker-proxy", "containerd-shim-runc"
]
return not any(exclusion in command for exclusion in exclusions)
def check_pid(pid):
return pid > 1000
def format_prometheus_metric(stat):
metrics = []
labels = f'pid="{stat["pid"]}", command="{stat["command"]}"'
metrics.append(f'exp_process_percent_usr{{{labels}}} {stat["percent_usr"]}')
metrics.append(f'exp_process_percent_system{{{labels}}} {stat["percent_system"]}')
metrics.append(f'exp_process_percent_cpu{{{labels}}} {stat["percent_cpu"]}')
metrics.append(f'exp_process_minflt_s{{{labels}}} {stat["minflt_s"]}')
metrics.append(f'exp_process_majflt_s{{{labels}}} {stat["majflt_s"]}')
metrics.append(f'exp_process_vsz{{{labels}}} {stat["vsz"]}')
metrics.append(f'exp_process_rss{{{labels}}} {stat["rss"]}')
metrics.append(f'exp_process_percent_mem{{{labels}}} {stat["percent_mem"]}')
return "\n".join(metrics)
def parse_java_process_output(java_process_output):
# Define regular expressions to match jar file and class name
jar_pattern = r'-jar\s+([^\s]+\.jar)'
class_pattern = r'-cp\s+[^ ]+\s+([a-zA-Z0-9.$]+)'
# Find the jar file name
jar_match = re.search(jar_pattern, java_process_output)
if jar_match:
return jar_match.group(1)
# Find the class name
class_match = re.search(class_pattern, java_process_output)
if class_match:
return class_match.group(1)
return None
def get_process_uptime(create_time):
now = datetime.now().timestamp()
return now - create_time
output_file_path = "/var/lib/prometheus/node-exporter/process_metrics.prom"
while True:
processes = psutil.process_iter(
['pid', 'name', 'cmdline', 'cpu_times', 'memory_info', 'cpu_percent', 'memory_percent', 'create_time'])
metrics = []
for proc in processes:
try:
with proc.oneshot():
pid = proc.info['pid']
command = ' '.join(proc.info['cmdline']) if proc.info['cmdline'] else proc.info['name']
cpu_times = proc.info['cpu_times']
memory_info = proc.info['memory_info']
percent_cpu = proc.info['cpu_percent']
percent_mem = proc.info['memory_percent']
create_time = proc.info['create_time']
uptime = get_process_uptime(create_time)
# skip short-lived processes
if uptime < 60:
continue
# skip processes with lower pids
if not check_pid(pid):
continue
if check_exclusions(command) and percent_mem > 0 and percent_cpu > 0:
stat = {
'pid': pid,
'command': command,
'percent_usr': cpu_times.user,
'percent_system': cpu_times.system,
'percent_cpu': percent_cpu,
'minflt_s': memory_info.minflt if hasattr(memory_info, 'minflt') else 0,
'majflt_s': memory_info.majflt if hasattr(memory_info, 'majflt') else 0,
'vsz': memory_info.vms,
'rss': memory_info.rss,
'percent_mem': percent_mem
}
if (('/' not in command and (percent_cpu > 3 or percent_mem > 3))
or '/' in command):
if 'java ' in command:
stat['command'] = parse_java_process_output(command)
elif 'python ' in command or 'node ' in command:
stat['command'] = os.path.basename(command.split(' ')[1])
elif ' ' in command:
stat['command'] = os.path.basename(command.split(' ')[0])
else:
stat['command'] = os.path.basename(command)
metrics.append(format_prometheus_metric(stat))
except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
pass
if len(metrics) > 0:
with open(output_file_path, 'w') as f:
f.write("\n".join(metrics) + "\n")
time.sleep(15)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment