GPU Monitoring with Cribl: Cribl Search Examples

gpu.json

{
  "name": "GPU Monitoring",
  "elements": [
    {
      "config": {
        "defaultValue": {
          "earliest": "-1h",
          "latest": "now",
          "timezone": "local"
        }
      },
      "id": "cpwnvz4yn",
      "inputId": "time_i",
      "type": "input.timerange",
      "layout": {
        "x": 0,
        "y": 0,
        "w": 2,
        "h": 2
      },
      "variant": "input",
      "title": "time_i"
    },
    {
      "config": {
        "separator": true,
        "shouldApplyUserChartSettings": false,
        "customData": {
          "stack": true,
          "seriesCount": 2
        },
        "legend": {
          "truncate": true
        }
      },
      "search": {
        "type": "inline",
        "earliest": "$time_i.earliest$",
        "latest": "$time_i.latest$",
        "query": "dataset=\"gpustats\" | timestats memory_free=avg(memory_free), memory_used=avg(memory_used)",
        "timezone": "$time_i.timezone$"
      },
      "id": "ttdr93pqz",
      "type": "chart.column",
      "layout": {
        "x": 0,
        "y": 0,
        "w": 12,
        "h": 4
      },
      "title": "Memory Usage"
    },
    {
      "config": {
        "label": "Power Draw Trends Over Time by Host"
      },
      "search": {
        "type": "inline",
        "earliest": "$time_i.earliest$",
        "latest": "$time_i.latest$",
        "query": "cribl dataset=gpustats | timestats span=5m avg_power_draw=avg(power_draw) by host",
        "timezone": "$time_i.timezone$"
      },
      "id": "uefrkryoq",
      "type": "chart.line",
      "layout": {
        "x": 0,
        "y": 4,
        "w": 6,
        "h": 4
      },
      "title": "Power Draw Trends Over Time by Host"
    },
    {
      "config": {
        "label": "Top 5 GPUs by Average Utilization"
      },
      "search": {
        "type": "inline",
        "earliest": "$time_i.earliest$",
        "latest": "$time_i.latest$",
        "query": "cribl dataset=gpustats | summarize utilization_avg=avg(utilization_gpu) by name | sort by utilization_avg desc | take 5",
        "timezone": "$time_i.timezone$"
      },
      "id": "f2r43q9cf",
      "type": "chart.pie",
      "layout": {
        "x": 6,
        "y": 4,
        "w": 6,
        "h": 4
      },
      "title": "Top 5 GPUs by Average Utilization"
    },
    {
      "config": {
        "separator": true,
        "shouldApplyUserChartSettings": false,
        "seriesInfo": {
          "temperature_celsius": {
            "color": "#00cccc"
          }
        },
        "yAxis": {
          "splitLine": true
        },
        "legend": {
          "truncate": true
        }
      },
      "search": {
        "type": "inline",
        "earliest": "$time_i.earliest$",
        "latest": "$time_i.latest$",
        "query": "dataset=gpustats | timestats temperature_celsius=avg(temperature_gpu)",
        "timezone": "$time_i.timezone$"
      },
      "id": "qj7qlt8na",
      "type": "chart.column",
      "layout": {
        "x": 0,
        "y": 8,
        "w": 12,
        "h": 4
      },
      "title": "Temperature"
    }
  ],
  "disabled": false
}