{ "__inputs": [ { "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "Prometheus datasource scraping vLLM, node-exporter, the GPU exporter and cAdvisor", "type": "datasource", "pluginId": "prometheus", "pluginName": "Prometheus" } ], "id": null, "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "panels": [], "title": "Overview", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "fixedColor": "green", "mode": "fixed" }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 4, "w": 3, "x": 0, "y": 1 }, "options": { "colorMode": "value", "graphMode": "none", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "count(up{job=\"node\",node=~\"$node\"} == 1)", "refId": "A" } ], "title": "Nodes selected", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "End-to-end tokens/sec per request — the SAME definition bench.sh reports (~40). Tokens-per-request divided by p50 end-to-end latency. NaN when idle. NOTE: both sides need sum() because request_success_total carries a finished_reason label that generation_tokens_total lacks; without sum() the vector division matches nothing and silently returns NaN.", "fieldConfig": { "defaults": { "color": { "fixedColor": "green", "mode": "fixed" }, "noValue": "idle", "unit": "none" }, "overrides": [] }, "gridPos": { "h": 4, "w": 3, "x": 3, "y": 1 }, "options": { "colorMode": "value", "graphMode": "area", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "(sum(rate(vllm:generation_tokens_total[5m])) / clamp_min(sum(rate(vllm:request_success_total[5m])),1e-9)) / clamp_min(histogram_quantile(0.5, sum(rate(vllm:e2e_request_latency_seconds_bucket[5m])) by (le)),1e-9)", "refId": "A" } ], "title": "Decode speed (tok/s, end-to-end)", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "unit": "none" }, "overrides": [] }, "gridPos": { "h": 4, "w": 3, "x": 6, "y": 1 }, "options": { "colorMode": "value", "graphMode": "area", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(vllm:num_requests_running)", "refId": "A" } ], "title": "Requests running", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 0.8 }, { "color": "red", "value": 0.95 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 4, "w": 3, "x": 9, "y": 1 }, "options": { "colorMode": "value", "graphMode": "area", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "max(vllm:kv_cache_usage_perc)", "refId": "A" } ], "title": "KV cache used", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 75 }, { "color": "red", "value": 85 } ] }, "unit": "celsius" }, "overrides": [] }, "gridPos": { "h": 4, "w": 3, "x": 12, "y": 1 }, "options": { "colorMode": "value", "graphMode": "area", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "max(nvidia_smi_temperature_gpu{node=~\"$node\"})", "refId": "A" } ], "title": "Hottest GPU", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "unit": "none" }, "overrides": [] }, "gridPos": { "h": 4, "w": 3, "x": 15, "y": 1 }, "options": { "colorMode": "value", "graphMode": "none", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "count(count by (name,node) (container_last_seen{name!=\"\",node=~\"$node\"}))", "refId": "A" } ], "title": "Containers up", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, "gridPos": { "h": 4, "w": 3, "x": 18, "y": 1 }, "options": { "colorMode": "value", "graphMode": "none", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(vllm:generation_tokens_total)", "refId": "A" } ], "title": "Total tokens generated", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Net saving over the selected range: what these tokens would have cost on Claude Opus 5 ($5/$25 per 1M, converted at 3.7103 PLN/USD) minus our electricity at Kraków G11 rates (1.04 zł/kWh). Uses increase() over the dashboard's selected time range, which IS correct for a single total-over-range figure (unlike a time series).", "fieldConfig": { "defaults": { "color": { "fixedColor": "green", "mode": "fixed" }, "decimals": 2, "unit": "suffix:zł" }, "overrides": [] }, "gridPos": { "h": 4, "w": 3, "x": 21, "y": 1 }, "options": { "colorMode": "value", "graphMode": "area", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "(((sum(increase(vllm:prompt_tokens_total[$__range]))/1e6)*5 + (sum(increase(vllm:generation_tokens_total[$__range]))/1e6)*25)*3.7103) - (240/1000)*1.04*($__range_s/3600)", "refId": "A" } ], "title": "Saved vs Claude Opus 5", "type": "stat" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, "panels": [], "title": "Alerts", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 0, "y": 6 }, "options": { "colorMode": "background", "graphMode": "none", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "count(ALERTS{alertstate=\"firing\"})", "refId": "A" } ], "title": "Firing alerts", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 1 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 4, "y": 6 }, "options": { "colorMode": "background", "graphMode": "none", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "count(ALERTS{alertstate=\"pending\"})", "refId": "A" } ], "title": "Pending alerts", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Firing Prometheus alerts. Empty table = nothing wrong. Rules live in ~/monitoring/rules/alerts.yml; edit then `docker restart prometheus`.", "fieldConfig": { "defaults": { "custom": { "align": "auto" } }, "overrides": [] }, "gridPos": { "h": 8, "w": 16, "x": 8, "y": 6 }, "options": { "showHeader": true }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "ALERTS{alertstate=\"firing\"}", "format": "table", "instant": true, "refId": "A" } ], "title": "Active alerts", "type": "table" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Which alerts fired and when. Flat/empty is the healthy state.", "fieldConfig": { "defaults": { "custom": { "fillOpacity": 25, "lineWidth": 1, "stacking": { "mode": "normal" } }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 6, "w": 24, "x": 0, "y": 14 }, "options": { "legend": { "calcs": [ "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (alertname) (ALERTS{alertstate=\"firing\"})", "legendFormat": "{{alertname}}", "refId": "A" } ], "title": "Alert activity over time", "type": "timeseries" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 20 }, "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Net saving over the selected range: what these tokens would have cost on Claude Opus 5 ($5/$25 per 1M, converted at 3.7103 PLN/USD) minus our electricity at Kraków G11 rates (1.04 zł/kWh). Uses increase() over the dashboard's selected time range, which IS correct for a single total-over-range figure (unlike a time series).", "fieldConfig": { "defaults": { "color": { "fixedColor": "green", "mode": "fixed" }, "decimals": 2, "unit": "suffix:zł" }, "overrides": [] }, "gridPos": { "h": 5, "w": 6, "x": 0, "y": 21 }, "options": { "colorMode": "value", "graphMode": "area", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "(((sum(increase(vllm:prompt_tokens_total[$__range]))/1e6)*5 + (sum(increase(vllm:generation_tokens_total[$__range]))/1e6)*25)*3.7103) - (240/1000)*1.04*($__range_s/3600)", "refId": "A" } ], "title": "Saved vs Claude Opus 5", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Versus GPT-6 Astra ($10/$50 per 1M), minus Kraków electricity. Uses increase() over the dashboard's selected time range, which IS correct for a single total-over-range figure (unlike a time series).", "fieldConfig": { "defaults": { "color": { "fixedColor": "green", "mode": "fixed" }, "decimals": 2, "unit": "suffix:zł" }, "overrides": [] }, "gridPos": { "h": 5, "w": 6, "x": 6, "y": 21 }, "options": { "colorMode": "value", "graphMode": "area", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "(((sum(increase(vllm:prompt_tokens_total[$__range]))/1e6)*10 + (sum(increase(vllm:generation_tokens_total[$__range]))/1e6)*50)*3.7103) - (240/1000)*1.04*($__range_s/3600)", "refId": "A" } ], "title": "Saved vs GPT-6 Astra", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Versus Claude Sonnet 5 ($2/$10 per 1M), minus Kraków electricity. Uses increase() over the dashboard's selected time range, which IS correct for a single total-over-range figure (unlike a time series).", "fieldConfig": { "defaults": { "color": { "fixedColor": "blue", "mode": "fixed" }, "decimals": 2, "unit": "suffix:zł" }, "overrides": [] }, "gridPos": { "h": 5, "w": 6, "x": 12, "y": 21 }, "options": { "colorMode": "value", "graphMode": "area", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "(((sum(increase(vllm:prompt_tokens_total[$__range]))/1e6)*2 + (sum(increase(vllm:generation_tokens_total[$__range]))/1e6)*10)*3.7103) - (240/1000)*1.04*($__range_s/3600)", "refId": "A" } ], "title": "Saved vs Claude Sonnet 5", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Kraków / Tauron taryfa G11: 1.04 zł brutto per kWh (all-in 2026 average). ASSUMED draw 2 x 120 W — the GX10 has no wall-power sensor. Excludes the fixed opłata mocowa (~21 zł/month), which doesn't scale with use. Edit ~/monitoring/rules/cost.yml. Uses increase() over the dashboard's selected time range, which IS correct for a single total-over-range figure (unlike a time series).", "fieldConfig": { "defaults": { "color": { "fixedColor": "orange", "mode": "fixed" }, "decimals": 2, "unit": "suffix:zł" }, "overrides": [] }, "gridPos": { "h": 5, "w": 6, "x": 18, "y": 21 }, "options": { "colorMode": "value", "graphMode": "none", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "(240/1000)*1.04*($__range_s/3600)", "refId": "A" } ], "title": "Our electricity cost", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "TRUE cumulative: built from the raw counters, so the line only ever rises. The previous version used increase()[range], which is a TRAILING window — it fell again as traffic aged out (measured: 0.96 -> 0.00 -> 0.71 -> 0.00), despite being labelled cumulative. NOTE: vLLM's counters reset when the container restarts, so this line drops to zero on every restart. The stat tiles above use increase()[range] which is correct for a single total-over-the-selected-range number.", "fieldConfig": { "defaults": { "custom": { "fillOpacity": 15, "lineWidth": 2 }, "unit": "suffix:zł" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 26 }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "((sum(vllm:prompt_tokens_total)/1e6)*5 + (sum(vllm:generation_tokens_total)/1e6)*25)*3.7103", "legendFormat": "Claude Opus 5", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "((sum(vllm:prompt_tokens_total)/1e6)*2 + (sum(vllm:generation_tokens_total)/1e6)*10)*3.7103", "legendFormat": "Claude Sonnet 5", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "((sum(vllm:prompt_tokens_total)/1e6)*1 + (sum(vllm:generation_tokens_total)/1e6)*5)*3.7103", "legendFormat": "Claude Haiku 4.5", "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "((sum(vllm:prompt_tokens_total)/1e6)*10 + (sum(vllm:generation_tokens_total)/1e6)*50)*3.7103", "legendFormat": "GPT-6 Astra", "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "((sum(vllm:prompt_tokens_total)/1e6)*2 + (sum(vllm:generation_tokens_total)/1e6)*12)*3.7103", "legendFormat": "GPT-5.6 Terra", "refId": "E" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "((sum(vllm:prompt_tokens_total)/1e6)*0.2 + (sum(vllm:generation_tokens_total)/1e6)*1.2)*3.7103", "legendFormat": "GPT-5.6 Luna", "refId": "F" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "(240/1000)*1.04*((time() - min(process_start_time_seconds{job=\"vllm\"}))/3600)", "legendFormat": "our electricity (Krakow G11)", "refId": "G" } ], "title": "Cumulative cost avoided (since vLLM start)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "zł per hour we would be burning at the current request rate, against our electricity draw. Flat zero when idle.", "fieldConfig": { "defaults": { "custom": { "fillOpacity": 15, "lineWidth": 2 }, "unit": "suffix:zł" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 26 }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "gx10:api_cost_rate_pln_per_hour", "legendFormat": "{{vendor}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "gx10:electricity_rate_pln_per_hour", "legendFormat": "our electricity", "refId": "B" } ], "title": "Spend rate being avoided", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "The basis for every cost figure above. increase() is used so vLLM restarts don't reset the numbers.", "fieldConfig": { "defaults": { "custom": { "fillOpacity": 15, "lineWidth": 2 }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 0, "y": 34 }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(increase(vllm:prompt_tokens_total[$__range]))/1e6", "legendFormat": "input Mtok", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(increase(vllm:generation_tokens_total[$__range]))/1e6", "legendFormat": "output Mtok", "refId": "B" } ], "title": "Tokens served (millions, in range)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Blended price per million tokens: hosted equivalent versus what we actually pay in electricity.", "fieldConfig": { "defaults": { "custom": { "fillOpacity": 8, "lineWidth": 2 }, "unit": "suffix:zł" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 12, "y": 34 }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "((((sum(increase(vllm:prompt_tokens_total[$__range]))/1e6)*5 + (sum(increase(vllm:generation_tokens_total[$__range]))/1e6)*25)*3.7103)) / clamp_min((sum(increase(vllm:prompt_tokens_total[$__range]))/1e6)+(sum(increase(vllm:generation_tokens_total[$__range]))/1e6),0.000001)", "legendFormat": "Claude Opus 5 equivalent", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "((240/1000)*1.04*($__range_s/3600)) / clamp_min((sum(increase(vllm:prompt_tokens_total[$__range]))/1e6)+(sum(increase(vllm:generation_tokens_total[$__range]))/1e6),0.000001)", "legendFormat": "our actual (electricity)", "refId": "B" } ], "title": "Effective zł per 1M tokens (blended)", "type": "timeseries" } ], "title": "Cost & savings vs hosted APIs (PLN, Kraków)", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 21 }, "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "2 x 20,000 PLN. Devices only — excludes the NAS, cables and the 200G DACs. Edit in ~/monitoring/rules/amortisation.yml", "fieldConfig": { "defaults": { "color": { "fixedColor": "text", "mode": "fixed" }, "decimals": 2, "unit": "suffix:zł" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 0, "y": 22 }, "options": { "colorMode": "value", "graphMode": "none", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "gx10:capex_pln", "refId": "A" } ], "title": "Hardware capex", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Claude Opus 5 equivalent cost of every token we served, minus electricity actually consumed, since 2026-09-10.", "fieldConfig": { "defaults": { "color": { "fixedColor": "blue", "mode": "thresholds" }, "decimals": 2, "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "orange", "value": 0 }, { "color": "green", "value": 1000 } ] }, "unit": "suffix:zł" }, "overrides": [] }, "gridPos": { "h": 4, "w": 5, "x": 4, "y": 22 }, "options": { "colorMode": "value", "graphMode": "none", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "gx10:net_saved_pln{vendor=\"claude-opus-5\"}", "refId": "A" } ], "title": "Net saved to date", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Fraction of the 40,000 zl capex recovered so far, versus Claude Opus 5 pricing.", "fieldConfig": { "defaults": { "decimals": 3, "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "orange", "value": 0.25 }, { "color": "yellow", "value": 0.5 }, { "color": "green", "value": 1 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 4, "w": 5, "x": 9, "y": 22 }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "gx10:amortised_fraction{vendor=\"claude-opus-5\"}", "refId": "A" } ], "title": "Amortised", "type": "gauge" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Trailing 7-day average. NEGATIVE means electricity currently costs more than the API usage we avoided — i.e. not amortising at all at this volume.", "fieldConfig": { "defaults": { "color": { "fixedColor": "blue", "mode": "thresholds" }, "decimals": 2, "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "orange", "value": 0 }, { "color": "green", "value": 50 } ] }, "unit": "suffix:zł/day" }, "overrides": [] }, "gridPos": { "h": 4, "w": 5, "x": 14, "y": 22 }, "options": { "colorMode": "value", "graphMode": "none", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "gx10:net_saved_pln_per_day_7d{vendor=\"claude-opus-5\"}", "refId": "A" } ], "title": "Net saving rate (7d)", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Years to recover the capex AT THE TRAILING 7-DAY RATE. With little traffic this is astronomically large and effectively means 'never at this usage'. It only becomes meaningful once real daily workload exists.", "fieldConfig": { "defaults": { "color": { "fixedColor": "blue", "mode": "thresholds" }, "decimals": 1, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 3 }, { "color": "orange", "value": 6 }, { "color": "red", "value": 15 } ] }, "unit": "suffix: yr" }, "overrides": [] }, "gridPos": { "h": 4, "w": 5, "x": 19, "y": 22 }, "options": { "colorMode": "value", "graphMode": "none", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "gx10:days_to_payoff{vendor=\"claude-opus-5\"} / 365", "refId": "A" } ], "title": "Projected payoff", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Payoff is where a 'net saved' line crosses the flat capex line.", "fieldConfig": { "defaults": { "custom": { "fillOpacity": 12, "lineWidth": 2 }, "unit": "suffix:zł" }, "overrides": [] }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 26 }, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "gx10:net_saved_pln{vendor=\"claude-opus-5\"}", "legendFormat": "net saved (vs Claude Opus 5)", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "gx10:avoided_pln_todate{vendor=\"gpt-6-astra\"} - on() group_left() gx10:electricity_spent_pln", "legendFormat": "net saved (vs GPT-6 Astra)", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "gx10:electricity_spent_pln", "legendFormat": "electricity spent", "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "gx10:capex_pln", "legendFormat": "capex target (40,000 zl)", "refId": "D" } ], "title": "Amortisation progress vs 40,000 zl capex", "type": "timeseries" }, { "gridPos": { "h": 16, "w": 24, "x": 0, "y": 34 }, "options": { "content": "### When does 40,000 zł pay for itself?\n\nLive projection above uses your **actual trailing 7-day** usage. Because the cluster is\nmostly idle today, that figure is meaningless until real workload exists. These scenarios\nare the useful guide:\n\n| Output Mtok/day | ≈ generation h/day | Net zł/day | Payoff |\n|---:|---:|---:|:--|\n| 0.10 | 0.7 | 5.14 | **21.3 yr** |\n| 0.25 | 1.7 | 21.84 | **5.0 yr** |\n| 0.50 | 3.5 | 49.66 | **2.2 yr** |\n| 1.00 | 6.9 | 105.32 | **1.0 yr** |\n| 2.00 | 13.9 | 216.63 | **185 days** |\n| 4.00 | 27.8* | 439.25 | **91 days** |\n\n\\* 27.8 h/day is impossible single-stream at 40 tok/s — but reachable at ~115 tok/s\naggregate (4 concurrent agents ≈ 9.7 h/day).\n\n**Assumptions — change these and the answer changes a lot:**\n- Baseline is **Claude Opus 5** ($5 in / $25 out per 1M, at 3.7103 PLN/USD).\n- **10:1 input:output** ratio, with API-side **prompt caching at 10%** of input price.\n Without caching the savings roughly double; assuming caching is the conservative choice.\n- Electricity **5.99 zł/day** (240 W at 1.04 zł/kWh) — an *assumption*, the GX10 exposes no\n wall-power sensor.\n- Capex is **devices only**: excludes NAS, cables and the 200G DACs.\n- Ignores resale value, and any cost for your own time.\n\n**Reality check:** versus cheap models (Haiku 4.5, GPT-5.6 Luna) this hardware may never pay\nback — those are cheaper per token than the electricity. The business case rests on\ndisplacing *frontier-tier* usage.", "mode": "markdown" }, "title": "Payoff scenarios & assumptions", "type": "text" } ], "title": "Amortisation — hardware payoff", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 22 }, "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Three legitimate but DIFFERENT numbers, measured on the same 512-token request:\n- end-to-end 40.0 tok/s = tokens / total request time (what bench.sh reports)\n- inter-token 57.1 tok/s = 1/time-per-output-token, excludes the ~0.08 s prefill\n- cluster 9.3 tok/s = rate()[1m], averages in IDLE time between requests\nIf you want 'how fast does it generate', read end-to-end. Blank/NaN here means the cluster was IDLE — histogram percentiles are undefined with zero requests. Cross-check 'Request rate & preemptions'; a flat zero there confirms idle, not a fault.", "fieldConfig": { "defaults": { "custom": { "fillOpacity": 18, "lineWidth": 2, "spanNulls": 600000 }, "noValue": "idle — no requests in window", "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 23 }, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "(sum(rate(vllm:generation_tokens_total[5m])) / clamp_min(sum(rate(vllm:request_success_total[5m])),1e-9)) / clamp_min(histogram_quantile(0.5, sum(rate(vllm:e2e_request_latency_seconds_bucket[5m])) by (le)),1e-9)", "legendFormat": "end-to-end tok/s (matches bench.sh ~40)", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "1 / histogram_quantile(0.5, sum(rate(vllm:request_time_per_output_token_seconds_bucket[5m])) by (le))", "legendFormat": "inter-token tok/s (1/TPOT, excl. prefill ~57)", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(vllm:generation_tokens_total[1m])", "legendFormat": "cluster output tok/s (incl. idle ~9)", "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(vllm:prompt_tokens_total[1m])", "legendFormat": "cluster prefill tok/s (incl. idle)", "refId": "D" } ], "title": "Throughput & decode speed", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { "fillOpacity": 15, "lineWidth": 2 }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 12, "y": 23 }, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "vllm:num_requests_running", "legendFormat": "running", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "vllm:num_requests_waiting", "legendFormat": "waiting", "refId": "B" } ], "title": "Requests running / waiting", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { "fillOpacity": 15, "lineWidth": 2 }, "max": 1, "min": 0, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 23 }, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "vllm:kv_cache_usage_perc", "legendFormat": "KV used", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(vllm:prefix_cache_hits_total[5m]) / clamp_min(rate(vllm:prefix_cache_queries_total[5m]),1)", "legendFormat": "prefix hit rate", "refId": "B" } ], "title": "KV cache & prefix hit rate", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "A latency percentile is undefined when no requests occurred — Prometheus returns NaN and the line breaks. Gaps here mean the cluster was IDLE, not that scraping failed. Cross-check with 'Request rate' below. Blank/NaN here means the cluster was IDLE — histogram percentiles are undefined with zero requests. Cross-check 'Request rate & preemptions'; a flat zero there confirms idle, not a fault.", "fieldConfig": { "defaults": { "custom": { "fillOpacity": 6, "lineWidth": 2, "pointSize": 6, "showPoints": "auto", "spanNulls": 600000 }, "noValue": "idle — no requests in window", "unit": "s" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 30 }, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.5, sum(rate(vllm:time_to_first_token_seconds_bucket[10m])) by (le))", "legendFormat": "TTFT p50", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.95, sum(rate(vllm:time_to_first_token_seconds_bucket[10m])) by (le))", "legendFormat": "TTFT p95", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.5, sum(rate(vllm:e2e_request_latency_seconds_bucket[10m])) by (le))", "legendFormat": "e2e p50", "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.95, sum(rate(vllm:e2e_request_latency_seconds_bucket[10m])) by (le))", "legendFormat": "e2e p95", "refId": "D" } ], "title": "Latency percentiles", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "A latency percentile is undefined when no requests occurred — Prometheus returns NaN and the line breaks. Gaps here mean the cluster was IDLE, not that scraping failed. Cross-check with 'Request rate' below. Blank/NaN here means the cluster was IDLE — histogram percentiles are undefined with zero requests. Cross-check 'Request rate & preemptions'; a flat zero there confirms idle, not a fault.", "fieldConfig": { "defaults": { "custom": { "fillOpacity": 6, "lineWidth": 2, "pointSize": 6, "showPoints": "auto", "spanNulls": 600000 }, "noValue": "idle — no requests in window", "unit": "s" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 12, "y": 30 }, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.5, sum(rate(vllm:request_time_per_output_token_seconds_bucket[10m])) by (le))", "legendFormat": "p50", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.95, sum(rate(vllm:request_time_per_output_token_seconds_bucket[10m])) by (le))", "legendFormat": "p95", "refId": "B" } ], "title": "Time per output token (p50/p95)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Use this to interpret gaps in the latency panels: flat zero here means the cluster was idle.", "fieldConfig": { "defaults": { "custom": { "fillOpacity": 15, "lineWidth": 2 }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 30 }, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(vllm:time_to_first_token_seconds_count[5m]))", "legendFormat": "requests/s", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(vllm:request_success_total[5m]))", "legendFormat": "completed/s", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(vllm:num_preemptions_total[5m])", "legendFormat": "preemptions/s", "refId": "C" } ], "title": "Request rate & preemptions", "type": "timeseries" } ], "title": "vLLM — tokens, latency, cache", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 }, "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { "fillOpacity": 10, "lineWidth": 2 }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 75 }, { "color": "red", "value": 85 } ] }, "unit": "celsius" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 24 }, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "nvidia_smi_temperature_gpu{node=~\"$node\"}", "legendFormat": "{{node}}", "refId": "A" } ], "title": "GPU temperature", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { "fillOpacity": 15, "lineWidth": 2 }, "unit": "watt" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 24 }, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "nvidia_smi_power_draw_watts{node=~\"$node\"}", "legendFormat": "{{node}}", "refId": "A" } ], "title": "GPU power draw", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { "fillOpacity": 15, "lineWidth": 2 }, "max": 1, "min": 0, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 24 }, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "nvidia_smi_utilization_gpu_ratio{node=~\"$node\"}", "legendFormat": "{{node}} sm", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "nvidia_smi_utilization_memory_ratio{node=~\"$node\"}", "legendFormat": "{{node}} mem", "refId": "B" } ], "title": "GPU utilization", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { "fillOpacity": 8, "lineWidth": 2 }, "unit": "hertz" }, "overrides": [] }, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 31 }, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "nvidia_smi_clocks_current_sm_clock_hz{node=~\"$node\"}", "legendFormat": "{{node}}", "refId": "A" } ], "title": "GPU SM clock", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { "fillOpacity": 20, "lineWidth": 2 }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 31 }, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "nvidia_smi_clocks_event_reasons_hw_thermal_slowdown{node=~\"$node\"}", "legendFormat": "{{node}} hw thermal", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "nvidia_smi_clocks_event_reasons_sw_power_cap{node=~\"$node\"}", "legendFormat": "{{node}} sw power cap", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "nvidia_smi_clocks_event_reasons_hw_power_brake_slowdown{node=~\"$node\"}", "legendFormat": "{{node}} hw power brake", "refId": "C" } ], "title": "Throttle reasons", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { "fillOpacity": 0, "lineWidth": 2 }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 31 }, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "nvidia_smi_pcie_link_gen_current{node=~\"$node\"}", "legendFormat": "{{node}} pcie gen", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "nvidia_smi_pcie_link_width_current{node=~\"$node\"}", "legendFormat": "{{node}} pcie width", "refId": "B" } ], "title": "GPU fan / PCIe link", "type": "timeseries" } ], "title": "GPU — per node", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 24 }, "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { "fillOpacity": 15, "lineWidth": 2 }, "unit": "pps" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 25 }, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(gx10_rdma_port_packets_total{direction=\"port_rcv\",node=~\"$node\"}[1m])", "legendFormat": "{{node}} {{device}} rx", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(gx10_rdma_port_packets_total{direction=\"port_xmit\",node=~\"$node\"}[1m])", "legendFormat": "{{node}} {{device}} tx", "refId": "B" } ], "title": "RDMA packet rate", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Any rise here means fabric trouble — link errors, discards or buffer overruns will silently cost throughput.", "fieldConfig": { "defaults": { "custom": { "fillOpacity": 20, "lineWidth": 2 }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 25 }, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(gx10_rdma_port_errors_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} {{device}} {{type}}", "refId": "A" } ], "title": "RDMA errors (should stay flat at 0)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { "fillOpacity": 0, "lineWidth": 2 }, "max": 5, "min": 0, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 25 }, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "gx10_rdma_port_state{node=~\"$node\"}", "legendFormat": "{{node}} {{device}}", "refId": "A" } ], "title": "RDMA link state (4 = ACTIVE)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "percentage_used is the SSD's own wear estimate (100 = rated endurance consumed). available_spare dropping below its threshold is an early failure signal.", "fieldConfig": { "defaults": { "custom": { "fillOpacity": 15, "lineWidth": 2 }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 32 }, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "gx10_nvme_health{attribute=\"temperature_celsius\",node=~\"$node\"}", "legendFormat": "{{node}} {{device}} temp", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "gx10_nvme_health{attribute=\"percentage_used_percent\",node=~\"$node\"}", "legendFormat": "{{node}} {{device}} wear %", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "gx10_nvme_health{attribute=\"available_spare_percent\",node=~\"$node\"}", "legendFormat": "{{node}} {{device}} spare %", "refId": "C" } ], "title": "NVMe temperature & wear", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "media_errors or critical_warning above 0 means the drive is failing — replace it.", "fieldConfig": { "defaults": { "custom": { "fillOpacity": 0, "lineWidth": 2 }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 32 }, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "gx10_nvme_health{attribute=\"media_errors\",node=~\"$node\"}", "legendFormat": "{{node}} media errors", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "gx10_nvme_health{attribute=\"critical_warning\",node=~\"$node\"}", "legendFormat": "{{node}} critical warning", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "gx10_nvme_health{attribute=\"error_log_entries\",node=~\"$node\"}", "legendFormat": "{{node}} error log", "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "gx10_nvme_health{attribute=\"unsafe_shutdowns\",node=~\"$node\"}", "legendFormat": "{{node}} unsafe shutdowns", "refId": "D" } ], "title": "NVMe errors & unsafe shutdowns", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Model checkpoints are 122-131 GB each; this tracks how much endurance staging them consumes.", "fieldConfig": { "defaults": { "custom": { "fillOpacity": 0, "lineWidth": 2 }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 32 }, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "gx10_nvme_bytes_total{direction=\"written\",node=~\"$node\"}", "legendFormat": "{{node}} written", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "gx10_nvme_bytes_total{direction=\"read\",node=~\"$node\"}", "legendFormat": "{{node}} read", "refId": "B" } ], "title": "NVMe lifetime bytes written", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "ConnectX-7 ASIC, NVMe and board sensors. NOTE: the GX10 exposes NO fan tachometers — fan RPM is not readable from the OS.", "fieldConfig": { "defaults": { "custom": { "fillOpacity": 6, "lineWidth": 2 }, "unit": "celsius" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 39 }, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "gx10_hwmon_temp_celsius{node=~\"$node\"}", "legendFormat": "{{node}} {{chip}} {{iface}} {{sensor}}", "refId": "A" } ], "title": "Component temperatures (NIC / NVMe / board)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Derived, not measured: vLLM's own estimated_read_bytes_per_gpu metric is present but never populates for MiniMaxM2 (always 0). This uses tok/s x ~2.65 GB read per node per token (10B active params at 4-bit, split TP=2) against the GB10's ~273 GB/s. Model-specific — recompute the constant if you change models. Low values here mean you are latency-bound, not bandwidth-bound.", "fieldConfig": { "defaults": { "custom": { "fillOpacity": 15, "lineWidth": 2 }, "max": 1, "min": 0, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 12, "y": 39 }, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "(sum(rate(vllm:generation_tokens_total[1m])) * 2.65e9) / 273e9", "legendFormat": "fraction of ~273 GB/s ceiling", "refId": "A" } ], "title": "Memory-bandwidth utilisation (derived)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Output tokens per watt across the cluster — the metric to optimise if you care about running cost.", "fieldConfig": { "defaults": { "custom": { "fillOpacity": 10, "lineWidth": 2 }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 39 }, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(vllm:generation_tokens_total[5m])) / sum(nvidia_smi_power_draw_watts)", "legendFormat": "tokens/watt-second", "refId": "A" } ], "title": "Energy efficiency (tokens per watt)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Tokens processed per engine iteration. Low values at batch=1 are expected; rising with concurrency means batching is working. Blank/NaN here means the cluster was IDLE — histogram percentiles are undefined with zero requests. Cross-check 'Request rate & preemptions'; a flat zero there confirms idle, not a fault.", "fieldConfig": { "defaults": { "custom": { "fillOpacity": 6, "lineWidth": 2, "spanNulls": 600000 }, "noValue": "idle — no requests in window", "unit": "none" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 0, "y": 46 }, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.5, sum(rate(vllm:iteration_tokens_total_bucket[5m])) by (le))", "legendFormat": "p50 tokens/step", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.95, sum(rate(vllm:iteration_tokens_total_bucket[5m])) by (le))", "legendFormat": "p95 tokens/step", "refId": "B" } ], "title": "Batch efficiency (tokens per engine step)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "If GPU util is high while CPU pressure is low, you are GPU/interconnect bound (expected here). CPU pressure rising would indicate a host-side bottleneck.", "fieldConfig": { "defaults": { "custom": { "fillOpacity": 8, "lineWidth": 2 }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 12, "y": 46 }, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(node_pressure_cpu_waiting_seconds_total{node=~\"$node\"}[2m])", "legendFormat": "{{node}} cpu pressure", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "nvidia_smi_utilization_gpu_ratio{node=~\"$node\"}", "legendFormat": "{{node}} gpu util", "refId": "B" } ], "title": "CPU pressure vs GPU utilisation", "type": "timeseries" } ], "title": "Fabric, storage health & efficiency", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 25 }, "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { "align": "auto" } }, "overrides": [] }, "gridPos": { "h": 6, "w": 10, "x": 0, "y": 26 }, "options": { "showHeader": true }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "node_uname_info{node=~\"$node\"}", "format": "table", "instant": true, "legendFormat": "", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "node_os_info{node=~\"$node\"}", "format": "table", "instant": true, "legendFormat": "", "refId": "B" } ], "title": "Host info", "type": "table" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { "fillOpacity": 0, "lineWidth": 2 }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 6, "w": 5, "x": 10, "y": 26 }, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "(time() - node_boot_time_seconds{node=~\"$node\"})", "legendFormat": "{{node}}", "refId": "A" } ], "title": "Uptime", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { "fillOpacity": 6, "lineWidth": 2 }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 6, "w": 9, "x": 15, "y": 26 }, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "node_load1{node=~\"$node\"}", "legendFormat": "{{node}} 1m", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "node_load5{node=~\"$node\"}", "legendFormat": "{{node}} 5m", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "node_load15{node=~\"$node\"}", "legendFormat": "{{node}} 15m", "refId": "C" } ], "title": "Load average", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { "fillOpacity": 30, "lineWidth": 2, "stacking": { "mode": "normal" } }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 32 }, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (node,mode) (rate(node_cpu_seconds_total{mode!=\"idle\",node=~\"$node\"}[2m])) / on(node) group_left count by (node) (node_cpu_seconds_total{mode=\"idle\",node=~\"$node\"})", "legendFormat": "{{node}} {{mode}}", "refId": "A" } ], "title": "CPU utilization by mode", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { "fillOpacity": 15, "lineWidth": 2 }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 12, "y": 32 }, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "node_memory_MemTotal_bytes{node=~\"$node\"} - node_memory_MemAvailable_bytes{node=~\"$node\"}", "legendFormat": "{{node}} used", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "node_memory_MemTotal_bytes{node=~\"$node\"}", "legendFormat": "{{node}} total", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "node_memory_Cached_bytes{node=~\"$node\"}", "legendFormat": "{{node}} cached", "refId": "C" } ], "title": "Memory", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { "fillOpacity": 10, "lineWidth": 2 }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 32 }, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(node_pressure_cpu_waiting_seconds_total{node=~\"$node\"}[2m])", "legendFormat": "{{node}} cpu", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(node_pressure_io_waiting_seconds_total{node=~\"$node\"}[2m])", "legendFormat": "{{node}} io", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(node_pressure_memory_waiting_seconds_total{node=~\"$node\"}[2m])", "legendFormat": "{{node}} mem", "refId": "C" } ], "title": "Pressure stall (PSI)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { "fillOpacity": 8, "lineWidth": 2 }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 6, "w": 6, "x": 0, "y": 39 }, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "node_procs_running{node=~\"$node\"}", "legendFormat": "{{node}} running", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "node_procs_blocked{node=~\"$node\"}", "legendFormat": "{{node}} blocked", "refId": "B" } ], "title": "Processes", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { "fillOpacity": 6, "lineWidth": 2 }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 6, "w": 6, "x": 6, "y": 39 }, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(node_context_switches_total{node=~\"$node\"}[2m])", "legendFormat": "{{node}} ctx/s", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(node_forks_total{node=~\"$node\"}[2m])", "legendFormat": "{{node}} forks/s", "refId": "B" } ], "title": "Context switches / forks", "type": "timeseries" } ], "title": "OS — CPU, memory & processes", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 26 }, "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { "fillOpacity": 15, "lineWidth": 2 }, "unit": "Bps" }, "overrides": [] }, "gridPos": { "h": 6, "w": 6, "x": 0, "y": 27 }, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(node_disk_read_bytes_total{device=~\"nvme.*\",node=~\"$node\"}[2m])", "legendFormat": "{{node}} {{device}} read", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(node_disk_written_bytes_total{device=~\"nvme.*\",node=~\"$node\"}[2m])", "legendFormat": "{{node}} {{device}} write", "refId": "B" } ], "title": "Disk I/O", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Real filesystems only (ext4/xfs/btrfs). node-exporter strips the --path.rootfs prefix, so the root fs is reported as '/', not '/host'.", "fieldConfig": { "defaults": { "custom": { "fillOpacity": 10, "lineWidth": 2 }, "max": 100, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 75 }, { "color": "red", "value": 90 } ] }, "unit": "percent" }, "overrides": [] }, "gridPos": { "h": 6, "w": 6, "x": 6, "y": 27 }, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "100 - (node_filesystem_avail_bytes{fstype=~\"ext4|xfs|btrfs\",node=~\"$node\"} / node_filesystem_size_bytes{fstype=~\"ext4|xfs|btrfs\",node=~\"$node\"} * 100)", "legendFormat": "{{node}} {{mountpoint}} %used", "refId": "A" } ], "title": "Filesystem usage", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "RoCE bypasses the kernel network stack, so node_network_* counters show almost nothing during inference (measured: 120 KB on netdev vs 1.46 GB on RDMA for the same load). These come from /sys/class/infiniband via the textfile exporter.", "fieldConfig": { "defaults": { "custom": { "fillOpacity": 12, "lineWidth": 2 }, "unit": "bps" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 12, "y": 27 }, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(gx10_rdma_port_data_bytes_total{direction=\"port_rcv\",node=~\"$node\"}[1m])*8", "legendFormat": "{{node}} {{device}} rx", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(gx10_rdma_port_data_bytes_total{direction=\"port_xmit\",node=~\"$node\"}[1m])*8", "legendFormat": "{{node}} {{device}} tx", "refId": "B" } ], "title": "200G fabric — RDMA traffic (real)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { "fillOpacity": 12, "lineWidth": 2 }, "unit": "bps" }, "overrides": [] }, "gridPos": { "h": 6, "w": 6, "x": 0, "y": 33 }, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(node_network_receive_bytes_total{device=\"enP7s7\",node=~\"$node\"}[1m])*8", "legendFormat": "{{node}} rx", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(node_network_transmit_bytes_total{device=\"enP7s7\",node=~\"$node\"}[1m])*8", "legendFormat": "{{node}} tx", "refId": "B" } ], "title": "Network — management LAN", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { "fillOpacity": 0, "lineWidth": 2 }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 6, "w": 6, "x": 6, "y": 33 }, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "node_thermal_zone_temp{node=~\"$node\"}", "legendFormat": "{{node}} {{type}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "node_filefd_allocated{node=~\"$node\"}", "legendFormat": "{{node}} fds", "refId": "B" } ], "title": "CPU thermal zones / file descriptors", "type": "timeseries" } ], "title": "OS — disk & network", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 27 }, "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { "fillOpacity": 30, "lineWidth": 2, "stacking": { "mode": "normal" } }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 }, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (node,name) (rate(container_cpu_usage_seconds_total{name!=\"\",node=~\"$node\"}[2m]))", "legendFormat": "{{node}}/{{name}}", "refId": "A" } ], "title": "Container CPU", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { "fillOpacity": 30, "lineWidth": 2, "stacking": { "mode": "normal" } }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 }, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (node,name) (container_memory_working_set_bytes{name!=\"\",node=~\"$node\"})", "legendFormat": "{{node}}/{{name}}", "refId": "A" } ], "title": "Container memory (RSS)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { "fillOpacity": 10, "lineWidth": 2 }, "unit": "Bps" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 36 }, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (node,name) (rate(container_network_receive_bytes_total{name!=\"\",node=~\"$node\"}[2m]))", "legendFormat": "{{node}}/{{name}} rx", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (node,name) (rate(container_network_transmit_bytes_total{name!=\"\",node=~\"$node\"}[2m]))", "legendFormat": "{{node}}/{{name}} tx", "refId": "B" } ], "title": "Container network RX/TX", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { "fillOpacity": 10, "lineWidth": 2 }, "unit": "Bps" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 12, "y": 36 }, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (node,name) (rate(container_fs_reads_bytes_total{name!=\"\",node=~\"$node\"}[2m]))", "legendFormat": "{{node}}/{{name}} read", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (node,name) (rate(container_fs_writes_bytes_total{name!=\"\",node=~\"$node\"}[2m]))", "legendFormat": "{{node}}/{{name}} write", "refId": "B" } ], "title": "Container disk I/O", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { "fillOpacity": 0, "lineWidth": 2 }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 36 }, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (node,name) (container_tasks_state{state=\"running\",name!=\"\",node=~\"$node\"})", "legendFormat": "{{node}}/{{name}} tasks", "refId": "A" } ], "title": "Container restarts / processes", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { "align": "auto" } }, "overrides": [] }, "gridPos": { "h": 9, "w": 24, "x": 0, "y": 43 }, "options": { "showHeader": true }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "topk(50, sum by (node,name,image) (container_last_seen{name!=\"\",node=~\"$node\"}))", "format": "table", "instant": true, "legendFormat": "", "refId": "A" } ], "title": "Container inventory", "type": "table" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Headroom for model checkpoints (each MiniMax build is ~122-131 GB on local NVMe).", "fieldConfig": { "defaults": { "custom": { "fillOpacity": 10, "lineWidth": 2 }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 6, "w": 6, "x": 0, "y": 52 }, "options": { "legend": { "calcs": [ "min", "lastNotNull" ], "displayMode": "table", "placement": "bottom" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "node_filesystem_avail_bytes{fstype=~\"ext4|xfs|btrfs\",node=~\"$node\"}", "legendFormat": "{{node}} {{mountpoint}} free", "refId": "A" } ], "title": "Filesystem free space", "type": "timeseries" } ], "title": "Docker containers", "type": "row" } ], "refresh": "10s", "schemaVersion": 39, "tags": [ "gx10", "vllm" ], "templating": { "list": [ { "allValue": ".*", "current": { "text": [ "All" ], "value": [ "$__all" ] }, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "includeAll": true, "label": "Node", "multi": true, "name": "node", "query": { "query": "label_values(up{job=\"node\"}, node)", "refId": "A" }, "refresh": 1, "sort": 1, "type": "query" } ] }, "time": { "from": "now-1h", "to": "now" }, "timezone": "browser", "title": "GX10 Cluster", "uid": "gx10-cluster", "version": 21 }