Last active
October 10, 2025 03:25
-
-
Save initcron/02f3842ce6911dd7e3224800bfbecf1e to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "annotations": { | |
| "list": [ | |
| { | |
| "builtIn": 1, | |
| "datasource": { | |
| "type": "grafana", | |
| "uid": "-- Grafana --" | |
| }, | |
| "enable": true, | |
| "hide": true, | |
| "iconColor": "rgba(0, 211, 255, 1)", | |
| "name": "Annotations & Alerts", | |
| "type": "dashboard" | |
| } | |
| ] | |
| }, | |
| "editable": true, | |
| "fiscalYearStartMonth": 0, | |
| "graphTooltip": 0, | |
| "id": 31, | |
| "links": [], | |
| "panels": [ | |
| { | |
| "fieldConfig": { | |
| "defaults": { | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": 0 | |
| }, | |
| { | |
| "color": "red", | |
| "value": 80 | |
| } | |
| ] | |
| } | |
| }, | |
| "overrides": [] | |
| }, | |
| "gridPos": { | |
| "h": 4, | |
| "w": 6, | |
| "x": 0, | |
| "y": 0 | |
| }, | |
| "id": 1, | |
| "options": { | |
| "colorMode": "value", | |
| "graphMode": "area", | |
| "justifyMode": "auto", | |
| "orientation": "auto", | |
| "percentChangeColorMode": "standard", | |
| "reduceOptions": { | |
| "calcs": [ | |
| "lastNotNull" | |
| ], | |
| "fields": "", | |
| "values": false | |
| }, | |
| "showPercentChange": false, | |
| "textMode": "auto", | |
| "wideLayout": true | |
| }, | |
| "pluginVersion": "12.1.1", | |
| "targets": [ | |
| { | |
| "expr": "sum(rate(chat_requests_total[1m]))", | |
| "refId": "A" | |
| } | |
| ], | |
| "title": "Chat RPS", | |
| "type": "stat" | |
| }, | |
| { | |
| "fieldConfig": { | |
| "defaults": { | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": 0 | |
| }, | |
| { | |
| "color": "red", | |
| "value": 80 | |
| } | |
| ] | |
| } | |
| }, | |
| "overrides": [] | |
| }, | |
| "gridPos": { | |
| "h": 4, | |
| "w": 6, | |
| "x": 6, | |
| "y": 0 | |
| }, | |
| "id": 4, | |
| "options": { | |
| "colorMode": "value", | |
| "graphMode": "area", | |
| "justifyMode": "auto", | |
| "orientation": "auto", | |
| "percentChangeColorMode": "standard", | |
| "reduceOptions": { | |
| "calcs": [ | |
| "lastNotNull" | |
| ], | |
| "fields": "", | |
| "values": false | |
| }, | |
| "showPercentChange": false, | |
| "textMode": "auto", | |
| "wideLayout": true | |
| }, | |
| "pluginVersion": "12.1.1", | |
| "targets": [ | |
| { | |
| "expr": "sum(rate(retriever_requests_total[1m]))", | |
| "refId": "A" | |
| } | |
| ], | |
| "title": "Retriever RPS", | |
| "type": "stat" | |
| }, | |
| { | |
| "fieldConfig": { | |
| "defaults": { | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": 0 | |
| }, | |
| { | |
| "color": "red", | |
| "value": 80 | |
| } | |
| ] | |
| } | |
| }, | |
| "overrides": [] | |
| }, | |
| "gridPos": { | |
| "h": 4, | |
| "w": 6, | |
| "x": 12, | |
| "y": 0 | |
| }, | |
| "id": 12, | |
| "options": { | |
| "colorMode": "value", | |
| "graphMode": "area", | |
| "justifyMode": "auto", | |
| "orientation": "auto", | |
| "percentChangeColorMode": "standard", | |
| "reduceOptions": { | |
| "calcs": [ | |
| "lastNotNull" | |
| ], | |
| "fields": "", | |
| "values": false | |
| }, | |
| "showPercentChange": false, | |
| "textMode": "auto", | |
| "wideLayout": true | |
| }, | |
| "pluginVersion": "12.1.1", | |
| "targets": [ | |
| { | |
| "expr": "sum(vllm:num_requests_waiting)", | |
| "legendFormat": "waiting", | |
| "refId": "A" | |
| }, | |
| { | |
| "expr": "sum(vllm:num_requests_running)", | |
| "legendFormat": "running", | |
| "refId": "B" | |
| } | |
| ], | |
| "title": "vLLM Queue / Running", | |
| "type": "stat" | |
| }, | |
| { | |
| "datasource": { | |
| "type": "prometheus", | |
| "uid": "prometheus" | |
| }, | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "palette-classic" | |
| }, | |
| "custom": { | |
| "axisBorderShow": false, | |
| "axisCenteredZero": false, | |
| "axisColorMode": "text", | |
| "axisLabel": "", | |
| "axisPlacement": "auto", | |
| "barAlignment": 0, | |
| "barWidthFactor": 0.6, | |
| "drawStyle": "line", | |
| "fillOpacity": 0, | |
| "gradientMode": "none", | |
| "hideFrom": { | |
| "legend": false, | |
| "tooltip": false, | |
| "viz": false | |
| }, | |
| "insertNulls": false, | |
| "lineInterpolation": "linear", | |
| "lineWidth": 1, | |
| "pointSize": 5, | |
| "scaleDistribution": { | |
| "type": "linear" | |
| }, | |
| "showPoints": "auto", | |
| "spanNulls": false, | |
| "stacking": { | |
| "group": "A", | |
| "mode": "none" | |
| }, | |
| "thresholdsStyle": { | |
| "mode": "off" | |
| } | |
| }, | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": 0 | |
| }, | |
| { | |
| "color": "red", | |
| "value": 80 | |
| } | |
| ] | |
| } | |
| }, | |
| "overrides": [] | |
| }, | |
| "gridPos": { | |
| "h": 6, | |
| "w": 6, | |
| "x": 18, | |
| "y": 0 | |
| }, | |
| "id": 10, | |
| "options": { | |
| "legend": { | |
| "calcs": [], | |
| "displayMode": "list", | |
| "placement": "bottom", | |
| "showLegend": true | |
| }, | |
| "tooltip": { | |
| "hideZeros": false, | |
| "mode": "single", | |
| "sort": "none" | |
| } | |
| }, | |
| "pluginVersion": "12.1.1", | |
| "targets": [ | |
| { | |
| "expr": "sum(rate(vllm:prompt_tokens_total[5m]))", | |
| "legendFormat": "prompt t/s", | |
| "refId": "A" | |
| }, | |
| { | |
| "expr": "sum(rate(vllm:generation_tokens_total[5m]))", | |
| "legendFormat": "generation t/s", | |
| "refId": "B" | |
| } | |
| ], | |
| "title": "vLLM Tokens/sec", | |
| "type": "timeseries" | |
| }, | |
| { | |
| "fieldConfig": { | |
| "defaults": { | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": 0 | |
| }, | |
| { | |
| "color": "red", | |
| "value": 80 | |
| } | |
| ] | |
| } | |
| }, | |
| "overrides": [] | |
| }, | |
| "gridPos": { | |
| "h": 4, | |
| "w": 6, | |
| "x": 0, | |
| "y": 4 | |
| }, | |
| "id": 7, | |
| "options": { | |
| "colorMode": "value", | |
| "graphMode": "area", | |
| "justifyMode": "auto", | |
| "orientation": "auto", | |
| "percentChangeColorMode": "standard", | |
| "reduceOptions": { | |
| "calcs": [ | |
| "lastNotNull" | |
| ], | |
| "fields": "", | |
| "values": false | |
| }, | |
| "showPercentChange": false, | |
| "textMode": "auto", | |
| "wideLayout": true | |
| }, | |
| "pluginVersion": "12.1.1", | |
| "targets": [ | |
| { | |
| "expr": "chat_prompt_tokens", | |
| "refId": "A" | |
| } | |
| ], | |
| "title": "Prompt Tokens (last)", | |
| "type": "stat" | |
| }, | |
| { | |
| "fieldConfig": { | |
| "defaults": { | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": 0 | |
| }, | |
| { | |
| "color": "red", | |
| "value": 80 | |
| } | |
| ] | |
| } | |
| }, | |
| "overrides": [] | |
| }, | |
| "gridPos": { | |
| "h": 4, | |
| "w": 6, | |
| "x": 6, | |
| "y": 4 | |
| }, | |
| "id": 8, | |
| "options": { | |
| "colorMode": "value", | |
| "graphMode": "area", | |
| "justifyMode": "auto", | |
| "orientation": "auto", | |
| "percentChangeColorMode": "standard", | |
| "reduceOptions": { | |
| "calcs": [ | |
| "lastNotNull" | |
| ], | |
| "fields": "", | |
| "values": false | |
| }, | |
| "showPercentChange": false, | |
| "textMode": "auto", | |
| "wideLayout": true | |
| }, | |
| "pluginVersion": "12.1.1", | |
| "targets": [ | |
| { | |
| "expr": "chat_completion_tokens", | |
| "refId": "A" | |
| } | |
| ], | |
| "title": "Completion Tokens (last)", | |
| "type": "stat" | |
| }, | |
| { | |
| "fieldConfig": { | |
| "defaults": { | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": 0 | |
| }, | |
| { | |
| "color": "red", | |
| "value": 80 | |
| } | |
| ] | |
| } | |
| }, | |
| "overrides": [] | |
| }, | |
| "gridPos": { | |
| "h": 4, | |
| "w": 6, | |
| "x": 12, | |
| "y": 4 | |
| }, | |
| "id": 9, | |
| "options": { | |
| "colorMode": "value", | |
| "graphMode": "area", | |
| "justifyMode": "auto", | |
| "orientation": "auto", | |
| "percentChangeColorMode": "standard", | |
| "reduceOptions": { | |
| "calcs": [ | |
| "lastNotNull" | |
| ], | |
| "fields": "", | |
| "values": false | |
| }, | |
| "showPercentChange": false, | |
| "textMode": "auto", | |
| "wideLayout": true | |
| }, | |
| "pluginVersion": "12.1.1", | |
| "targets": [ | |
| { | |
| "expr": "chat_total_tokens", | |
| "refId": "A" | |
| } | |
| ], | |
| "title": "Total Tokens (last)", | |
| "type": "stat" | |
| }, | |
| { | |
| "datasource": { | |
| "type": "prometheus", | |
| "uid": "prometheus" | |
| }, | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "palette-classic" | |
| }, | |
| "custom": { | |
| "axisBorderShow": false, | |
| "axisCenteredZero": false, | |
| "axisColorMode": "text", | |
| "axisLabel": "", | |
| "axisPlacement": "auto", | |
| "barAlignment": 0, | |
| "barWidthFactor": 0.6, | |
| "drawStyle": "line", | |
| "fillOpacity": 0, | |
| "gradientMode": "none", | |
| "hideFrom": { | |
| "legend": false, | |
| "tooltip": false, | |
| "viz": false | |
| }, | |
| "insertNulls": false, | |
| "lineInterpolation": "linear", | |
| "lineWidth": 1, | |
| "pointSize": 5, | |
| "scaleDistribution": { | |
| "type": "linear" | |
| }, | |
| "showPoints": "auto", | |
| "spanNulls": false, | |
| "stacking": { | |
| "group": "A", | |
| "mode": "none" | |
| }, | |
| "thresholdsStyle": { | |
| "mode": "off" | |
| } | |
| }, | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": 0 | |
| }, | |
| { | |
| "color": "red", | |
| "value": 80 | |
| } | |
| ] | |
| } | |
| }, | |
| "overrides": [] | |
| }, | |
| "gridPos": { | |
| "h": 7, | |
| "w": 12, | |
| "x": 0, | |
| "y": 8 | |
| }, | |
| "id": 11, | |
| "options": { | |
| "legend": { | |
| "calcs": [], | |
| "displayMode": "list", | |
| "placement": "bottom", | |
| "showLegend": true | |
| }, | |
| "tooltip": { | |
| "hideZeros": false, | |
| "mode": "single", | |
| "sort": "none" | |
| } | |
| }, | |
| "pluginVersion": "12.1.1", | |
| "targets": [ | |
| { | |
| "expr": "histogram_quantile(0.95, sum by (le) (rate(vllm:time_to_first_token_seconds_bucket[5m])))", | |
| "legendFormat": "TTFT p95", | |
| "refId": "A" | |
| }, | |
| { | |
| "expr": "histogram_quantile(0.95, sum by (le) (rate(vllm:e2e_request_latency_seconds_bucket[5m])))", | |
| "legendFormat": "E2E p95", | |
| "refId": "B" | |
| }, | |
| { | |
| "expr": "histogram_quantile(0.95, sum by (le) (rate(vllm:request_inference_time_seconds_bucket[5m])))", | |
| "legendFormat": "inference p95", | |
| "refId": "C" | |
| }, | |
| { | |
| "expr": "histogram_quantile(0.95, sum by (le) (rate(vllm:time_per_output_token_seconds_bucket[5m])))", | |
| "legendFormat": "TPOT p95", | |
| "refId": "D" | |
| } | |
| ], | |
| "title": "vLLM Latencies (p95)", | |
| "type": "timeseries" | |
| }, | |
| { | |
| "fieldConfig": { | |
| "defaults": { | |
| "custom": { | |
| "align": "auto", | |
| "cellOptions": { | |
| "type": "auto" | |
| }, | |
| "inspect": false | |
| }, | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": 0 | |
| }, | |
| { | |
| "color": "red", | |
| "value": 80 | |
| } | |
| ] | |
| } | |
| }, | |
| "overrides": [] | |
| }, | |
| "gridPos": { | |
| "h": 7, | |
| "w": 6, | |
| "x": 12, | |
| "y": 8 | |
| }, | |
| "id": 13, | |
| "options": { | |
| "cellHeight": "sm", | |
| "footer": { | |
| "countRows": false, | |
| "fields": "", | |
| "reducer": [ | |
| "sum" | |
| ], | |
| "show": false | |
| }, | |
| "showHeader": true | |
| }, | |
| "pluginVersion": "12.1.1", | |
| "targets": [ | |
| { | |
| "expr": "retriever_index_items", | |
| "legendFormat": "index_items", | |
| "refId": "A" | |
| }, | |
| { | |
| "expr": "retriever_meta_items", | |
| "legendFormat": "meta_items", | |
| "refId": "B" | |
| } | |
| ], | |
| "title": "Retriever Index & Meta Sizes", | |
| "type": "table" | |
| }, | |
| { | |
| "datasource": { | |
| "type": "prometheus" | |
| }, | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "palette-classic" | |
| }, | |
| "custom": { | |
| "axisBorderShow": false, | |
| "axisCenteredZero": false, | |
| "axisColorMode": "text", | |
| "axisLabel": "", | |
| "axisPlacement": "auto", | |
| "barAlignment": 0, | |
| "barWidthFactor": 0.6, | |
| "drawStyle": "line", | |
| "fillOpacity": 0, | |
| "gradientMode": "none", | |
| "hideFrom": { | |
| "legend": false, | |
| "tooltip": false, | |
| "viz": false | |
| }, | |
| "insertNulls": false, | |
| "lineInterpolation": "linear", | |
| "lineWidth": 1, | |
| "pointSize": 5, | |
| "scaleDistribution": { | |
| "type": "linear" | |
| }, | |
| "showPoints": "auto", | |
| "spanNulls": false, | |
| "stacking": { | |
| "group": "A", | |
| "mode": "none" | |
| }, | |
| "thresholdsStyle": { | |
| "mode": "off" | |
| } | |
| }, | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": 0 | |
| }, | |
| { | |
| "color": "red", | |
| "value": 80 | |
| } | |
| ] | |
| } | |
| }, | |
| "overrides": [] | |
| }, | |
| "gridPos": { | |
| "h": 7, | |
| "w": 12, | |
| "x": 0, | |
| "y": 15 | |
| }, | |
| "id": 14, | |
| "interval": "1m", | |
| "options": { | |
| "legend": { | |
| "calcs": [], | |
| "displayMode": "list", | |
| "placement": "bottom", | |
| "showLegend": true | |
| }, | |
| "tooltip": { | |
| "hideZeros": false, | |
| "mode": "single", | |
| "sort": "none" | |
| } | |
| }, | |
| "pluginVersion": "12.1.1", | |
| "targets": [ | |
| { | |
| "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate5m{cluster=\"\", namespace=\"atharva-ml\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"\", namespace=\"atharva-ml\", workload_type=~\"deployment\"}\n) by (workload, workload_type)", | |
| "legendFormat": "{{workload}}", | |
| "refId": "A" | |
| } | |
| ], | |
| "title": "Deployments CPU (atharva-ml)", | |
| "type": "timeseries" | |
| }, | |
| { | |
| "datasource": { | |
| "type": "prometheus", | |
| "uid": "prometheus" | |
| }, | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "palette-classic" | |
| }, | |
| "custom": { | |
| "axisBorderShow": false, | |
| "axisCenteredZero": false, | |
| "axisColorMode": "text", | |
| "axisLabel": "", | |
| "axisPlacement": "auto", | |
| "barAlignment": 0, | |
| "barWidthFactor": 0.6, | |
| "drawStyle": "line", | |
| "fillOpacity": 0, | |
| "gradientMode": "none", | |
| "hideFrom": { | |
| "legend": false, | |
| "tooltip": false, | |
| "viz": false | |
| }, | |
| "insertNulls": false, | |
| "lineInterpolation": "linear", | |
| "lineWidth": 1, | |
| "pointSize": 5, | |
| "scaleDistribution": { | |
| "type": "linear" | |
| }, | |
| "showPoints": "auto", | |
| "spanNulls": false, | |
| "stacking": { | |
| "group": "A", | |
| "mode": "none" | |
| }, | |
| "thresholdsStyle": { | |
| "mode": "off" | |
| } | |
| }, | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": 0 | |
| }, | |
| { | |
| "color": "red", | |
| "value": 80 | |
| } | |
| ] | |
| } | |
| }, | |
| "overrides": [] | |
| }, | |
| "gridPos": { | |
| "h": 7, | |
| "w": 12, | |
| "x": 12, | |
| "y": 15 | |
| }, | |
| "id": 15, | |
| "interval": "1m", | |
| "options": { | |
| "legend": { | |
| "calcs": [], | |
| "displayMode": "list", | |
| "placement": "bottom", | |
| "showLegend": true | |
| }, | |
| "tooltip": { | |
| "hideZeros": false, | |
| "mode": "single", | |
| "sort": "none" | |
| } | |
| }, | |
| "pluginVersion": "12.1.1", | |
| "targets": [ | |
| { | |
| "expr": "sum(\n container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"\", namespace=\"atharva-ml\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"\", namespace=\"atharva-ml\", workload_type=~\"deployment\"}\n) by (workload, workload_type)", | |
| "legendFormat": "{{workload}}", | |
| "refId": "A" | |
| } | |
| ], | |
| "title": "Deployments Memory (atharva-ml)", | |
| "type": "timeseries" | |
| }, | |
| { | |
| "datasource": { | |
| "type": "prometheus", | |
| "uid": "prometheus" | |
| }, | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "palette-classic" | |
| }, | |
| "custom": { | |
| "axisBorderShow": false, | |
| "axisCenteredZero": false, | |
| "axisColorMode": "text", | |
| "axisLabel": "", | |
| "axisPlacement": "auto", | |
| "barAlignment": 0, | |
| "barWidthFactor": 0.6, | |
| "drawStyle": "line", | |
| "fillOpacity": 0, | |
| "gradientMode": "none", | |
| "hideFrom": { | |
| "legend": false, | |
| "tooltip": false, | |
| "viz": false | |
| }, | |
| "insertNulls": false, | |
| "lineInterpolation": "linear", | |
| "lineWidth": 1, | |
| "pointSize": 5, | |
| "scaleDistribution": { | |
| "type": "linear" | |
| }, | |
| "showPoints": "auto", | |
| "spanNulls": false, | |
| "stacking": { | |
| "group": "A", | |
| "mode": "none" | |
| }, | |
| "thresholdsStyle": { | |
| "mode": "off" | |
| } | |
| }, | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": 0 | |
| }, | |
| { | |
| "color": "red", | |
| "value": 80 | |
| } | |
| ] | |
| } | |
| }, | |
| "overrides": [] | |
| }, | |
| "gridPos": { | |
| "h": 7, | |
| "w": 12, | |
| "x": 0, | |
| "y": 22 | |
| }, | |
| "id": 2, | |
| "options": { | |
| "legend": { | |
| "calcs": [], | |
| "displayMode": "list", | |
| "placement": "bottom", | |
| "showLegend": true | |
| }, | |
| "tooltip": { | |
| "hideZeros": false, | |
| "mode": "single", | |
| "sort": "none" | |
| } | |
| }, | |
| "pluginVersion": "12.1.1", | |
| "targets": [ | |
| { | |
| "expr": "histogram_quantile(0.50, sum by (le) (rate(chat_end_to_end_latency_seconds_bucket[5m])))", | |
| "legendFormat": "p50", | |
| "refId": "A" | |
| }, | |
| { | |
| "expr": "histogram_quantile(0.95, sum by (le) (rate(chat_end_to_end_latency_seconds_bucket[5m])))", | |
| "legendFormat": "p95", | |
| "refId": "B" | |
| } | |
| ], | |
| "title": "Chat E2E Latency (p50/p95)", | |
| "type": "timeseries" | |
| }, | |
| { | |
| "datasource": { | |
| "type": "prometheus", | |
| "uid": "prometheus" | |
| }, | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "palette-classic" | |
| }, | |
| "custom": { | |
| "axisBorderShow": false, | |
| "axisCenteredZero": false, | |
| "axisColorMode": "text", | |
| "axisLabel": "", | |
| "axisPlacement": "auto", | |
| "barAlignment": 0, | |
| "barWidthFactor": 0.6, | |
| "drawStyle": "line", | |
| "fillOpacity": 0, | |
| "gradientMode": "none", | |
| "hideFrom": { | |
| "legend": false, | |
| "tooltip": false, | |
| "viz": false | |
| }, | |
| "insertNulls": false, | |
| "lineInterpolation": "linear", | |
| "lineWidth": 1, | |
| "pointSize": 5, | |
| "scaleDistribution": { | |
| "type": "linear" | |
| }, | |
| "showPoints": "auto", | |
| "spanNulls": false, | |
| "stacking": { | |
| "group": "A", | |
| "mode": "none" | |
| }, | |
| "thresholdsStyle": { | |
| "mode": "off" | |
| } | |
| }, | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": 0 | |
| }, | |
| { | |
| "color": "red", | |
| "value": 80 | |
| } | |
| ] | |
| } | |
| }, | |
| "overrides": [] | |
| }, | |
| "gridPos": { | |
| "h": 7, | |
| "w": 12, | |
| "x": 12, | |
| "y": 22 | |
| }, | |
| "id": 3, | |
| "options": { | |
| "legend": { | |
| "calcs": [], | |
| "displayMode": "list", | |
| "placement": "bottom", | |
| "showLegend": true | |
| }, | |
| "tooltip": { | |
| "hideZeros": false, | |
| "mode": "single", | |
| "sort": "none" | |
| } | |
| }, | |
| "pluginVersion": "12.1.1", | |
| "targets": [ | |
| { | |
| "expr": "histogram_quantile(0.95, sum by (le) (rate(rag_retrieval_latency_seconds_bucket[5m])))", | |
| "legendFormat": "retriever (inside chat)", | |
| "refId": "A" | |
| }, | |
| { | |
| "expr": "histogram_quantile(0.95, sum by (le) (rate(vllm:e2e_request_latency_seconds_bucket[5m])))", | |
| "legendFormat": "vLLM (inside chat)", | |
| "refId": "B" | |
| } | |
| ], | |
| "title": "Chat Sub-steps p95 (Retriever & vLLM)", | |
| "type": "timeseries" | |
| }, | |
| { | |
| "datasource": { | |
| "type": "prometheus", | |
| "uid": "prometheus" | |
| }, | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "palette-classic" | |
| }, | |
| "custom": { | |
| "axisBorderShow": false, | |
| "axisCenteredZero": false, | |
| "axisColorMode": "text", | |
| "axisLabel": "", | |
| "axisPlacement": "auto", | |
| "barAlignment": 0, | |
| "barWidthFactor": 0.6, | |
| "drawStyle": "line", | |
| "fillOpacity": 0, | |
| "gradientMode": "none", | |
| "hideFrom": { | |
| "legend": false, | |
| "tooltip": false, | |
| "viz": false | |
| }, | |
| "insertNulls": false, | |
| "lineInterpolation": "linear", | |
| "lineWidth": 1, | |
| "pointSize": 5, | |
| "scaleDistribution": { | |
| "type": "linear" | |
| }, | |
| "showPoints": "auto", | |
| "spanNulls": false, | |
| "stacking": { | |
| "group": "A", | |
| "mode": "none" | |
| }, | |
| "thresholdsStyle": { | |
| "mode": "off" | |
| } | |
| }, | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": 0 | |
| }, | |
| { | |
| "color": "red", | |
| "value": 80 | |
| } | |
| ] | |
| } | |
| }, | |
| "overrides": [] | |
| }, | |
| "gridPos": { | |
| "h": 7, | |
| "w": 12, | |
| "x": 0, | |
| "y": 29 | |
| }, | |
| "id": 5, | |
| "options": { | |
| "legend": { | |
| "calcs": [], | |
| "displayMode": "list", | |
| "placement": "bottom", | |
| "showLegend": true | |
| }, | |
| "tooltip": { | |
| "hideZeros": false, | |
| "mode": "single", | |
| "sort": "none" | |
| } | |
| }, | |
| "pluginVersion": "12.1.1", | |
| "targets": [ | |
| { | |
| "expr": "histogram_quantile(0.50, sum by (le) (rate(retriever_search_latency_seconds_bucket[5m])))", | |
| "legendFormat": "p50", | |
| "refId": "A" | |
| }, | |
| { | |
| "expr": "histogram_quantile(0.95, sum by (le) (rate(retriever_search_latency_seconds_bucket[5m])))", | |
| "legendFormat": "p95", | |
| "refId": "B" | |
| } | |
| ], | |
| "title": "Retriever Search Latency (p50/p95)", | |
| "type": "timeseries" | |
| }, | |
| { | |
| "datasource": { | |
| "type": "prometheus", | |
| "uid": "prometheus" | |
| }, | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "palette-classic" | |
| }, | |
| "custom": { | |
| "axisBorderShow": false, | |
| "axisCenteredZero": false, | |
| "axisColorMode": "text", | |
| "axisLabel": "", | |
| "axisPlacement": "auto", | |
| "barAlignment": 0, | |
| "barWidthFactor": 0.6, | |
| "drawStyle": "line", | |
| "fillOpacity": 0, | |
| "gradientMode": "none", | |
| "hideFrom": { | |
| "legend": false, | |
| "tooltip": false, | |
| "viz": false | |
| }, | |
| "insertNulls": false, | |
| "lineInterpolation": "linear", | |
| "lineWidth": 1, | |
| "pointSize": 5, | |
| "scaleDistribution": { | |
| "type": "linear" | |
| }, | |
| "showPoints": "auto", | |
| "spanNulls": false, | |
| "stacking": { | |
| "group": "A", | |
| "mode": "none" | |
| }, | |
| "thresholdsStyle": { | |
| "mode": "off" | |
| } | |
| }, | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": 0 | |
| }, | |
| { | |
| "color": "red", | |
| "value": 80 | |
| } | |
| ] | |
| } | |
| }, | |
| "overrides": [] | |
| }, | |
| "gridPos": { | |
| "h": 7, | |
| "w": 12, | |
| "x": 12, | |
| "y": 29 | |
| }, | |
| "id": 6, | |
| "options": { | |
| "legend": { | |
| "calcs": [], | |
| "displayMode": "list", | |
| "placement": "bottom", | |
| "showLegend": true | |
| }, | |
| "tooltip": { | |
| "hideZeros": false, | |
| "mode": "single", | |
| "sort": "none" | |
| } | |
| }, | |
| "pluginVersion": "12.1.1", | |
| "targets": [ | |
| { | |
| "expr": "histogram_quantile(0.95, sum by (le) (rate(retriever_dense_encode_latency_seconds_bucket[5m])))", | |
| "legendFormat": "dense encode", | |
| "refId": "A" | |
| }, | |
| { | |
| "expr": "histogram_quantile(0.95, sum by (le) (rate(retriever_dense_faiss_latency_seconds_bucket[5m])))", | |
| "legendFormat": "dense faiss", | |
| "refId": "B" | |
| }, | |
| { | |
| "expr": "histogram_quantile(0.95, sum by (le) (rate(retriever_sparse_vectorize_latency_seconds_bucket[5m])))", | |
| "legendFormat": "sparse vectorize", | |
| "refId": "C" | |
| }, | |
| { | |
| "expr": "histogram_quantile(0.95, sum by (le) (rate(retriever_sparse_dot_latency_seconds_bucket[5m])))", | |
| "legendFormat": "sparse dot", | |
| "refId": "D" | |
| } | |
| ], | |
| "title": "Retriever Sub-steps p95 (Dense/Sparse)", | |
| "type": "timeseries" | |
| } | |
| ], | |
| "preload": false, | |
| "schemaVersion": 41, | |
| "tags": [], | |
| "templating": { | |
| "list": [] | |
| }, | |
| "time": { | |
| "from": "now-15m", | |
| "to": "now" | |
| }, | |
| "timepicker": {}, | |
| "timezone": "browser", | |
| "title": "Atharva LLMOps- Overview", | |
| "uid": "09dbc849-df80-4b82-9a49-7cd6bb2b7cad", | |
| "version": 1 | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment