Stéphane Tailland
3937b4f366
Deploys the whole stack into the observability namespace of the local cluster with Kustomize alone — no helm install, no dashboard created by hand in the UI. The Makefile carries every command: deploy, dashboards, diagnostics, PVC flush, teardown. Dashboards are files, never UI objects. Each carries a stable uid, which is what keeps URLs, cross-links and alerts alive across redeployments, and references datasources by stable uid rather than by name. The dashboard ConfigMap is generated without a name-suffix hash so applying it updates it in place and Grafana reloads without a restart, while the provisioning ConfigMaps keep their hash precisely so that a change there does force one. Prometheus scrapes the kubelet /metrics/resource endpoint rather than /metrics/cadvisor: on this cluster cadvisor exposes machine_* series only and no container_* at all, which would leave every CPU and memory panel empty. Alloy ships every pod's logs to Loki through the Kubernetes API, so no /var/log mount is needed. Tempo's metrics_generator feeds span metrics and the service graph back into Prometheus, and the datasources are wired so a log line's trace_id opens the matching trace. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
1773 lines
42 KiB
JSON
1773 lines
42 KiB
JSON
{
|
|
"uid": "obs-librechat",
|
|
"title": "LibreChat — Overview",
|
|
"description": "HTTP traffic, Node.js runtime, MongoDB access and logs for LibreChat. Built-in /metrics endpoint, scraped with a Bearer token.",
|
|
"tags": [
|
|
"observability",
|
|
"librechat",
|
|
"application",
|
|
"metrics",
|
|
"logs"
|
|
],
|
|
"editable": false,
|
|
"schemaVersion": 39,
|
|
"version": 1,
|
|
"refresh": "30s",
|
|
"time": {
|
|
"from": "now-6h",
|
|
"to": "now"
|
|
},
|
|
"timezone": "browser",
|
|
"templating": {
|
|
"list": []
|
|
},
|
|
"panels": [
|
|
{
|
|
"id": 1,
|
|
"type": "row",
|
|
"title": "Overview",
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 0
|
|
},
|
|
"panels": []
|
|
},
|
|
{
|
|
"id": 2,
|
|
"type": "stat",
|
|
"title": "Requests / s",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 6,
|
|
"x": 0,
|
|
"y": 1
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum(rate(http_requests_total{job=\"librechat\"}[$__rate_interval]))",
|
|
"instant": false
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "reqps",
|
|
"mappings": [],
|
|
"decimals": 2,
|
|
"color": {
|
|
"mode": "fixed",
|
|
"fixedColor": "blue"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "blue",
|
|
"value": null
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"textMode": "auto",
|
|
"justifyMode": "auto"
|
|
}
|
|
},
|
|
{
|
|
"id": 3,
|
|
"type": "stat",
|
|
"title": "Error rate",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 6,
|
|
"x": 6,
|
|
"y": 1
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "(sum(rate(http_requests_total{job=\"librechat\",status=~\"5..\"}[$__rate_interval])) or vector(0)) / clamp_min(sum(rate(http_requests_total{job=\"librechat\"}[$__rate_interval])), 0.001)",
|
|
"instant": false
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percentunit",
|
|
"mappings": [],
|
|
"decimals": 2,
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "orange",
|
|
"value": 0.01
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 0.05
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"textMode": "auto",
|
|
"justifyMode": "auto"
|
|
},
|
|
"description": "5xx only: 401/403 are expected on authenticated routes and would pollute the indicator. `or vector(0)` is required: with no errors at all the numerator is an empty vector, and empty/x stays empty -- the panel would read \"No data\" instead of 0%. clamp_min covers the zero-traffic case."
|
|
},
|
|
{
|
|
"id": 4,
|
|
"type": "stat",
|
|
"title": "Latency p95",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 6,
|
|
"x": 12,
|
|
"y": 1
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "histogram_quantile(0.95, sum by (le) (rate(http_request_duration_seconds_bucket{job=\"librechat\"}[$__rate_interval])))",
|
|
"instant": false
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "s",
|
|
"mappings": [],
|
|
"decimals": 3,
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "orange",
|
|
"value": 0.5
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 2
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"textMode": "auto",
|
|
"justifyMode": "auto"
|
|
},
|
|
"description": "Native prom-client histogram: covers every request, not just the traced ones."
|
|
},
|
|
{
|
|
"id": 5,
|
|
"type": "stat",
|
|
"title": "Generations in flight",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 6,
|
|
"x": 18,
|
|
"y": 1
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum(generation_jobs_in_flight{job=\"librechat\"})",
|
|
"instant": false
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"mappings": [],
|
|
"decimals": 0,
|
|
"color": {
|
|
"mode": "fixed",
|
|
"fixedColor": "purple"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "purple",
|
|
"value": null
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"textMode": "auto",
|
|
"justifyMode": "auto"
|
|
},
|
|
"description": "LLM completions in flight. Each holds a handle and heap for its whole duration."
|
|
},
|
|
{
|
|
"id": 6,
|
|
"type": "row",
|
|
"title": "HTTP traffic",
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 6
|
|
},
|
|
"panels": []
|
|
},
|
|
{
|
|
"id": 7,
|
|
"type": "timeseries",
|
|
"title": "Requests by status class",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 7
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum by (class) (label_replace(rate(http_requests_total{job=\"librechat\"}[$__rate_interval]), \"class\", \"$1xx\", \"status\", \"(.)..\"))",
|
|
"legendFormat": "{{class}}"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "reqps",
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 70,
|
|
"showPoints": "never",
|
|
"pointSize": 8,
|
|
"gradientMode": "none",
|
|
"spanNulls": false,
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"lineInterpolation": "linear",
|
|
"stacking": {
|
|
"mode": "normal",
|
|
"group": "A"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"min": 0
|
|
},
|
|
"overrides": [
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "2xx"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "green"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "3xx"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "blue"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "4xx"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "orange"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "5xx"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "red"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "SUCCESS"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "green"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "REDIRECTION"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "blue"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "CLIENT_ERROR"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "orange"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "SERVER_ERROR"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "red"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 8,
|
|
"type": "timeseries",
|
|
"title": "Request latency",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 7
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "histogram_quantile(0.50, sum by (le) (rate(http_request_duration_seconds_bucket{job=\"librechat\"}[$__rate_interval])))",
|
|
"legendFormat": "p50"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "histogram_quantile(0.95, sum by (le) (rate(http_request_duration_seconds_bucket{job=\"librechat\"}[$__rate_interval])))",
|
|
"legendFormat": "p95"
|
|
},
|
|
{
|
|
"refId": "C",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "histogram_quantile(0.99, sum by (le) (rate(http_request_duration_seconds_bucket{job=\"librechat\"}[$__rate_interval])))",
|
|
"legendFormat": "p99"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "s",
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 2,
|
|
"fillOpacity": 0,
|
|
"showPoints": "never",
|
|
"pointSize": 8,
|
|
"gradientMode": "none",
|
|
"spanNulls": false,
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"lineInterpolation": "smooth"
|
|
},
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"min": 0
|
|
},
|
|
"overrides": [
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "p50"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "light-blue"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "p95"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "blue"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "p99"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "dark-blue"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 9,
|
|
"type": "bargauge",
|
|
"title": "Busiest routes",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 15
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "topk(10, sum by (path) (rate(http_requests_total{job=\"librechat\"}[$__rate_interval])))",
|
|
"legendFormat": "{{path}}",
|
|
"instant": true
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "reqps",
|
|
"min": 0,
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "continuous-BlPu"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "blue",
|
|
"value": null
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"displayMode": "gradient",
|
|
"orientation": "horizontal",
|
|
"showUnfilled": true,
|
|
"valueMode": "text",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 10,
|
|
"type": "timeseries",
|
|
"title": "Requests in flight",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 15
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum(http_requests_in_flight{job=\"librechat\"})",
|
|
"legendFormat": "in flight"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 2,
|
|
"fillOpacity": 8,
|
|
"showPoints": "never",
|
|
"pointSize": 8,
|
|
"gradientMode": "none",
|
|
"spanNulls": false,
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"lineInterpolation": "smooth"
|
|
},
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"min": 0,
|
|
"decimals": 0
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": false,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"description": "Instantaneous concurrency. On a single-threaded runtime, a value that never comes back down points at a blocked event loop."
|
|
},
|
|
{
|
|
"id": 11,
|
|
"type": "row",
|
|
"title": "Node.js runtime",
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 23
|
|
},
|
|
"panels": []
|
|
},
|
|
{
|
|
"id": 12,
|
|
"type": "timeseries",
|
|
"title": "V8 heap",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 24
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum(nodejs_heap_size_used_bytes{job=\"librechat\"})",
|
|
"legendFormat": "used"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum(nodejs_heap_size_total_bytes{job=\"librechat\"})",
|
|
"legendFormat": "allocated"
|
|
},
|
|
{
|
|
"refId": "C",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum(process_resident_memory_bytes{job=\"librechat\"})",
|
|
"legendFormat": "process RSS"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "bytes",
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 2,
|
|
"fillOpacity": 8,
|
|
"showPoints": "never",
|
|
"pointSize": 8,
|
|
"gradientMode": "none",
|
|
"spanNulls": false,
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"lineInterpolation": "smooth"
|
|
},
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"min": 0
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"description": "Old-space is capped at 1200 MB by NODE_OPTIONS. Used heap flattening just below that precedes a V8 crash."
|
|
},
|
|
{
|
|
"id": 13,
|
|
"type": "timeseries",
|
|
"title": "Event loop lag",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 24
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum(nodejs_eventloop_lag_p50_seconds{job=\"librechat\"})",
|
|
"legendFormat": "p50"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum(nodejs_eventloop_lag_p99_seconds{job=\"librechat\"})",
|
|
"legendFormat": "p99"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "s",
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 2,
|
|
"fillOpacity": 0,
|
|
"showPoints": "never",
|
|
"pointSize": 8,
|
|
"gradientMode": "none",
|
|
"spanNulls": false,
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"lineInterpolation": "smooth"
|
|
},
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"min": 0,
|
|
"decimals": 4
|
|
},
|
|
"overrides": [
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "p50"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "light-blue"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "p99"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "dark-blue"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"description": "Event loop lag: the single best saturation signal for a Node process. Past roughly 100 ms, every request suffers."
|
|
},
|
|
{
|
|
"id": 14,
|
|
"type": "timeseries",
|
|
"title": "Time spent in GC",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 24
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum by (kind) (rate(nodejs_gc_duration_seconds_sum{job=\"librechat\"}[$__rate_interval]))",
|
|
"legendFormat": "{{kind}}"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "s",
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 2,
|
|
"fillOpacity": 8,
|
|
"showPoints": "never",
|
|
"pointSize": 8,
|
|
"gradientMode": "none",
|
|
"spanNulls": false,
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"lineInterpolation": "smooth"
|
|
},
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"min": 0
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"description": "GC seconds per elapsed second: directly the fraction of CPU time stolen from request handling."
|
|
},
|
|
{
|
|
"id": 15,
|
|
"type": "row",
|
|
"title": "MongoDB (Mongoose)",
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 32
|
|
},
|
|
"panels": []
|
|
},
|
|
{
|
|
"id": 16,
|
|
"type": "timeseries",
|
|
"title": "Mongo queries by operation",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 33
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "topk(10, sum by (model, operation) (rate(mongoose_queries_total{job=\"librechat\"}[$__rate_interval])))",
|
|
"legendFormat": "{{model}}.{{operation}}"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "reqps",
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 2,
|
|
"fillOpacity": 8,
|
|
"showPoints": "never",
|
|
"pointSize": 8,
|
|
"gradientMode": "none",
|
|
"spanNulls": false,
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"lineInterpolation": "smooth"
|
|
},
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"min": 0
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 17,
|
|
"type": "timeseries",
|
|
"title": "Mongo query latency",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 33
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "histogram_quantile(0.50, sum by (le) (rate(mongoose_query_duration_seconds_bucket{job=\"librechat\"}[$__rate_interval])))",
|
|
"legendFormat": "p50"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "histogram_quantile(0.95, sum by (le) (rate(mongoose_query_duration_seconds_bucket{job=\"librechat\"}[$__rate_interval])))",
|
|
"legendFormat": "p95"
|
|
},
|
|
{
|
|
"refId": "C",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "histogram_quantile(0.99, sum by (le) (rate(mongoose_query_duration_seconds_bucket{job=\"librechat\"}[$__rate_interval])))",
|
|
"legendFormat": "p99"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "s",
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 2,
|
|
"fillOpacity": 0,
|
|
"showPoints": "never",
|
|
"pointSize": 8,
|
|
"gradientMode": "none",
|
|
"spanNulls": false,
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"lineInterpolation": "smooth"
|
|
},
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"min": 0
|
|
},
|
|
"overrides": [
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "p50"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "light-blue"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "p95"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "blue"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "p99"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "dark-blue"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 18,
|
|
"type": "timeseries",
|
|
"title": "Failed Mongo queries",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 33
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum by (model) (rate(mongoose_queries_total{job=\"librechat\",status!=\"success\"}[$__rate_interval]))",
|
|
"legendFormat": "{{model}}"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "reqps",
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 2,
|
|
"fillOpacity": 0,
|
|
"showPoints": "never",
|
|
"pointSize": 8,
|
|
"gradientMode": "none",
|
|
"spanNulls": false,
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"lineInterpolation": "smooth"
|
|
},
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"min": 0
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"description": "Empty in nominal operation."
|
|
},
|
|
{
|
|
"id": 19,
|
|
"type": "row",
|
|
"title": "Logs",
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 40
|
|
},
|
|
"panels": []
|
|
},
|
|
{
|
|
"id": 20,
|
|
"type": "timeseries",
|
|
"title": "Log volume by level",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 41
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "sum by (detected_level) (count_over_time({namespace=\"librechat\",container=\"librechat\"}[$__auto]))",
|
|
"legendFormat": "{{detected_level}}",
|
|
"queryType": "range"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"min": 0,
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "bars",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 80,
|
|
"showPoints": "never",
|
|
"barAlignment": 0,
|
|
"stacking": {
|
|
"mode": "normal",
|
|
"group": "A"
|
|
},
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto"
|
|
}
|
|
},
|
|
"overrides": [
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "critical"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "dark-red"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "error"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "red"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "warn"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "orange"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "info"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "green"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "debug"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "blue"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "unknown"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "text"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"sum"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"description": "`detected_level` is inferred by Loki at ingestion, so one query works across every log format."
|
|
},
|
|
{
|
|
"id": 21,
|
|
"type": "timeseries",
|
|
"title": "Errors by container (whole namespace)",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 41
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "sum by (container) (count_over_time({namespace=\"librechat\"} | detected_level =~ \"error|critical\" [$__auto]))",
|
|
"legendFormat": "{{container}}",
|
|
"queryType": "range"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"min": 0,
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "bars",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 80,
|
|
"showPoints": "never",
|
|
"barAlignment": 0,
|
|
"stacking": {
|
|
"mode": "normal",
|
|
"group": "A"
|
|
},
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto"
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"sum"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"description": "Namespace-wide so the datastore's failures show up next to the application's."
|
|
},
|
|
{
|
|
"id": 22,
|
|
"type": "logs",
|
|
"title": "Recent errors and warnings",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"gridPos": {
|
|
"h": 11,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 49
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "{namespace=\"librechat\"} | detected_level =~ \"error|critical|warn\"",
|
|
"queryType": "range"
|
|
}
|
|
],
|
|
"options": {
|
|
"showTime": true,
|
|
"showLabels": false,
|
|
"showCommonLabels": false,
|
|
"wrapLogMessage": true,
|
|
"prettifyLogMessage": false,
|
|
"sortOrder": "Descending",
|
|
"dedupStrategy": "none",
|
|
"enableLogDetails": true
|
|
},
|
|
"description": "Click a line to expand it. Where the line carries a trace_id, the Loki datasource's derived field opens the matching trace in Tempo."
|
|
},
|
|
{
|
|
"id": 23,
|
|
"type": "row",
|
|
"title": "Container resources",
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 60
|
|
},
|
|
"panels": []
|
|
},
|
|
{
|
|
"id": 24,
|
|
"type": "timeseries",
|
|
"title": "CPU per container",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 61
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum by (pod, container) (rate(container_cpu_usage_seconds_total{namespace=\"librechat\",pod=~\"librechat-.*\",container!=\"\"}[$__rate_interval]))",
|
|
"legendFormat": "{{pod}}/{{container}}"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 2,
|
|
"fillOpacity": 8,
|
|
"showPoints": "never",
|
|
"pointSize": 8,
|
|
"gradientMode": "none",
|
|
"spanNulls": false,
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"lineInterpolation": "smooth"
|
|
},
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"min": 0,
|
|
"decimals": 3
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"description": "Cores consumed."
|
|
},
|
|
{
|
|
"id": 25,
|
|
"type": "timeseries",
|
|
"title": "Memory per container",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 61
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum by (pod, container) (container_memory_working_set_bytes{namespace=\"librechat\",pod=~\"librechat-.*\",container!=\"\"})",
|
|
"legendFormat": "{{pod}}/{{container}}"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "bytes",
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 2,
|
|
"fillOpacity": 8,
|
|
"showPoints": "never",
|
|
"pointSize": 8,
|
|
"gradientMode": "none",
|
|
"spanNulls": false,
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"lineInterpolation": "smooth"
|
|
},
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"min": 0
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"description": "Working set: what the kernel cannot reclaim under pressure. This is the figure the OOM killer compares against the limit."
|
|
},
|
|
{
|
|
"id": 26,
|
|
"type": "timeseries",
|
|
"title": "Restarts over the window",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 61
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum by (pod) (changes(container_start_time_seconds{namespace=\"librechat\",pod=~\"librechat-.*\",container!=\"\"}[$__range]))",
|
|
"legendFormat": "{{pod}}"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 2,
|
|
"fillOpacity": 0,
|
|
"showPoints": "never",
|
|
"pointSize": 8,
|
|
"gradientMode": "none",
|
|
"spanNulls": false,
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"lineInterpolation": "smooth"
|
|
},
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"min": 0,
|
|
"decimals": 0
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"description": "Container start count over the displayed window. Stays at 0 in nominal operation."
|
|
}
|
|
]
|
|
}
|