Stéphane Tailland
3937b4f366
Deploys the whole stack into the observability namespace of the local cluster with Kustomize alone — no helm install, no dashboard created by hand in the UI. The Makefile carries every command: deploy, dashboards, diagnostics, PVC flush, teardown. Dashboards are files, never UI objects. Each carries a stable uid, which is what keeps URLs, cross-links and alerts alive across redeployments, and references datasources by stable uid rather than by name. The dashboard ConfigMap is generated without a name-suffix hash so applying it updates it in place and Grafana reloads without a restart, while the provisioning ConfigMaps keep their hash precisely so that a change there does force one. Prometheus scrapes the kubelet /metrics/resource endpoint rather than /metrics/cadvisor: on this cluster cadvisor exposes machine_* series only and no container_* at all, which would leave every CPU and memory panel empty. Alloy ships every pod's logs to Loki through the Kubernetes API, so no /var/log mount is needed. Tempo's metrics_generator feeds span metrics and the service graph back into Prometheus, and the datasources are wired so a log line's trace_id opens the matching trace. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
1638 lines
39 KiB
JSON
1638 lines
39 KiB
JSON
{
|
|
"uid": "obs-keycloak",
|
|
"title": "Keycloak — Overview",
|
|
"description": "HTTP traffic, JVM runtime, connection pool and logs for Keycloak. Latencies come from Tempo span metrics: the Micrometer timer exposes no buckets.",
|
|
"tags": [
|
|
"observability",
|
|
"keycloak",
|
|
"application",
|
|
"metrics",
|
|
"logs"
|
|
],
|
|
"editable": false,
|
|
"schemaVersion": 39,
|
|
"version": 1,
|
|
"refresh": "30s",
|
|
"time": {
|
|
"from": "now-6h",
|
|
"to": "now"
|
|
},
|
|
"timezone": "browser",
|
|
"templating": {
|
|
"list": []
|
|
},
|
|
"panels": [
|
|
{
|
|
"id": 1,
|
|
"type": "row",
|
|
"title": "Overview",
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 0
|
|
},
|
|
"panels": []
|
|
},
|
|
{
|
|
"id": 2,
|
|
"type": "stat",
|
|
"title": "Requests / s",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 6,
|
|
"x": 0,
|
|
"y": 1
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum(rate(http_server_requests_seconds_count{namespace=\"keycloak\"}[$__rate_interval]))",
|
|
"instant": false
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "reqps",
|
|
"mappings": [],
|
|
"decimals": 2,
|
|
"color": {
|
|
"mode": "fixed",
|
|
"fixedColor": "blue"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "blue",
|
|
"value": null
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"textMode": "auto",
|
|
"justifyMode": "auto"
|
|
}
|
|
},
|
|
{
|
|
"id": 3,
|
|
"type": "stat",
|
|
"title": "Error rate",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 6,
|
|
"x": 6,
|
|
"y": 1
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "(sum(rate(http_server_requests_seconds_count{namespace=\"keycloak\",outcome=~\"CLIENT_ERROR|SERVER_ERROR\"}[$__rate_interval])) or vector(0)) / clamp_min(sum(rate(http_server_requests_seconds_count{namespace=\"keycloak\"}[$__rate_interval])), 0.001)",
|
|
"instant": false
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percentunit",
|
|
"mappings": [],
|
|
"decimals": 2,
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "orange",
|
|
"value": 0.01
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 0.05
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"textMode": "auto",
|
|
"justifyMode": "auto"
|
|
},
|
|
"description": "Share of 4xx and 5xx responses. `or vector(0)` is required: with no errors at all the numerator is an empty vector, and empty/x stays empty -- the panel would read \"No data\" instead of 0%. clamp_min covers the zero-traffic case."
|
|
},
|
|
{
|
|
"id": 4,
|
|
"type": "stat",
|
|
"title": "Latency p95",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 6,
|
|
"x": 12,
|
|
"y": 1
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "histogram_quantile(0.95, sum by (le) (rate(traces_spanmetrics_latency_bucket{service=\"keycloak\",span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval])))",
|
|
"instant": false
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "s",
|
|
"mappings": [],
|
|
"decimals": 3,
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "orange",
|
|
"value": 0.5
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 2
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"textMode": "auto",
|
|
"justifyMode": "auto"
|
|
},
|
|
"description": "Source: span metrics generated by Tempo. Keycloak's Micrometer timer exposes neither buckets nor quantiles, so no percentile is computable from /metrics."
|
|
},
|
|
{
|
|
"id": 5,
|
|
"type": "stat",
|
|
"title": "JVM heap used",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 6,
|
|
"x": 18,
|
|
"y": 1
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum(jvm_memory_used_bytes{namespace=\"keycloak\",area=\"heap\"})",
|
|
"instant": false
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "bytes",
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "fixed",
|
|
"fixedColor": "purple"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "purple",
|
|
"value": null
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"textMode": "auto",
|
|
"justifyMode": "auto"
|
|
}
|
|
},
|
|
{
|
|
"id": 6,
|
|
"type": "row",
|
|
"title": "HTTP traffic",
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 6
|
|
},
|
|
"panels": []
|
|
},
|
|
{
|
|
"id": 7,
|
|
"type": "timeseries",
|
|
"title": "Requests by outcome",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 7
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum by (outcome) (rate(http_server_requests_seconds_count{namespace=\"keycloak\"}[$__rate_interval]))",
|
|
"legendFormat": "{{outcome}}"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "reqps",
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 70,
|
|
"showPoints": "never",
|
|
"pointSize": 8,
|
|
"gradientMode": "none",
|
|
"spanNulls": false,
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"lineInterpolation": "linear",
|
|
"stacking": {
|
|
"mode": "normal",
|
|
"group": "A"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"min": 0
|
|
},
|
|
"overrides": [
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "2xx"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "green"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "3xx"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "blue"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "4xx"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "orange"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "5xx"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "red"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "SUCCESS"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "green"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "REDIRECTION"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "blue"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "CLIENT_ERROR"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "orange"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "SERVER_ERROR"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "red"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"description": "Stacked: total height is throughput, status colours isolate the failing share."
|
|
},
|
|
{
|
|
"id": 8,
|
|
"type": "timeseries",
|
|
"title": "Latency (Tempo span metrics)",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 7
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "histogram_quantile(0.50, sum by (le) (rate(traces_spanmetrics_latency_bucket{service=\"keycloak\",span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval])))",
|
|
"legendFormat": "p50"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "histogram_quantile(0.95, sum by (le) (rate(traces_spanmetrics_latency_bucket{service=\"keycloak\",span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval])))",
|
|
"legendFormat": "p95"
|
|
},
|
|
{
|
|
"refId": "C",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "histogram_quantile(0.99, sum by (le) (rate(traces_spanmetrics_latency_bucket{service=\"keycloak\",span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval])))",
|
|
"legendFormat": "p99"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "s",
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 2,
|
|
"fillOpacity": 0,
|
|
"showPoints": "never",
|
|
"pointSize": 8,
|
|
"gradientMode": "none",
|
|
"spanNulls": false,
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"lineInterpolation": "smooth"
|
|
},
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"min": 0
|
|
},
|
|
"overrides": [
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "p50"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "light-blue"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "p95"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "blue"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "p99"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "dark-blue"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"description": "One hue from light to dark: p50/p95/p99 are ordered magnitudes of the same measure, not distinct entities."
|
|
},
|
|
{
|
|
"id": 9,
|
|
"type": "bargauge",
|
|
"title": "Busiest endpoints",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 15
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "topk(10, sum by (uri) (rate(http_server_requests_seconds_count{namespace=\"keycloak\"}[$__rate_interval])))",
|
|
"legendFormat": "{{uri}}",
|
|
"instant": true
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "reqps",
|
|
"min": 0,
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "continuous-BlPu"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "blue",
|
|
"value": null
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"displayMode": "gradient",
|
|
"orientation": "horizontal",
|
|
"showUnfilled": true,
|
|
"valueMode": "text",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 10,
|
|
"type": "timeseries",
|
|
"title": "Mean latency per endpoint",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 15
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "topk(10, sum by (uri) (rate(http_server_requests_seconds_sum{namespace=\"keycloak\"}[$__rate_interval])) / clamp_min(sum by (uri) (rate(http_server_requests_seconds_count{namespace=\"keycloak\"}[$__rate_interval])), 0.001))",
|
|
"legendFormat": "{{uri}}"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "s",
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 2,
|
|
"fillOpacity": 0,
|
|
"showPoints": "never",
|
|
"pointSize": 8,
|
|
"gradientMode": "none",
|
|
"spanNulls": false,
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"lineInterpolation": "smooth"
|
|
},
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"min": 0
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"description": "A mean, not a percentile: that is all a bucket-less timer allows. A mean hides the tail -- for the tail, see the p99 panel."
|
|
},
|
|
{
|
|
"id": 11,
|
|
"type": "row",
|
|
"title": "JVM runtime",
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 23
|
|
},
|
|
"panels": []
|
|
},
|
|
{
|
|
"id": 12,
|
|
"type": "timeseries",
|
|
"title": "JVM memory by area",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 24
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum by (area) (jvm_memory_used_bytes{namespace=\"keycloak\"})",
|
|
"legendFormat": "{{area}}"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum(jvm_memory_max_bytes{namespace=\"keycloak\",area=\"heap\"} > 0)",
|
|
"legendFormat": "max heap"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "bytes",
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 2,
|
|
"fillOpacity": 8,
|
|
"showPoints": "never",
|
|
"pointSize": 8,
|
|
"gradientMode": "none",
|
|
"spanNulls": false,
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"lineInterpolation": "smooth"
|
|
},
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"min": 0
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 13,
|
|
"type": "timeseries",
|
|
"title": "GC pauses",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 24
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum(rate(jvm_gc_pause_seconds_sum{namespace=\"keycloak\"}[$__rate_interval])) / clamp_min(sum(rate(jvm_gc_pause_seconds_count{namespace=\"keycloak\"}[$__rate_interval])), 0.001)",
|
|
"legendFormat": "mean duration"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "s",
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 2,
|
|
"fillOpacity": 8,
|
|
"showPoints": "never",
|
|
"pointSize": 8,
|
|
"gradientMode": "none",
|
|
"spanNulls": false,
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"lineInterpolation": "smooth"
|
|
},
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"min": 0,
|
|
"decimals": 4
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": false,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"description": "Mean GC pause duration. Drift here usually precedes a rise in p99 latency."
|
|
},
|
|
{
|
|
"id": 14,
|
|
"type": "timeseries",
|
|
"title": "JVM threads",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 24
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum(jvm_threads_live_threads{namespace=\"keycloak\"})",
|
|
"legendFormat": "live"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum(jvm_threads_daemon_threads{namespace=\"keycloak\"})",
|
|
"legendFormat": "daemon"
|
|
},
|
|
{
|
|
"refId": "C",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum(jvm_threads_peak_threads{namespace=\"keycloak\"})",
|
|
"legendFormat": "peak"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 2,
|
|
"fillOpacity": 0,
|
|
"showPoints": "never",
|
|
"pointSize": 8,
|
|
"gradientMode": "none",
|
|
"spanNulls": false,
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"lineInterpolation": "smooth"
|
|
},
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"min": 0
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 15,
|
|
"type": "row",
|
|
"title": "PostgreSQL connection pool (Agroal)",
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 32
|
|
},
|
|
"panels": []
|
|
},
|
|
{
|
|
"id": 16,
|
|
"type": "timeseries",
|
|
"title": "Pool connections",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 33
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum by (datasource) (agroal_active_count{namespace=\"keycloak\"})",
|
|
"legendFormat": "active"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum by (datasource) (agroal_available_count{namespace=\"keycloak\"})",
|
|
"legendFormat": "available"
|
|
},
|
|
{
|
|
"refId": "C",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum by (datasource) (agroal_max_used_count{namespace=\"keycloak\"})",
|
|
"legendFormat": "peak used"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 2,
|
|
"fillOpacity": 0,
|
|
"showPoints": "never",
|
|
"pointSize": 8,
|
|
"gradientMode": "none",
|
|
"spanNulls": false,
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"lineInterpolation": "smooth"
|
|
},
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"min": 0
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 17,
|
|
"type": "timeseries",
|
|
"title": "Waiting for a connection",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 33
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum(agroal_awaiting_count{namespace=\"keycloak\"})",
|
|
"legendFormat": "requests waiting"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum(agroal_blocking_time_average_milliseconds{namespace=\"keycloak\"})",
|
|
"legendFormat": "mean blocking time (ms)"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 2,
|
|
"fillOpacity": 0,
|
|
"showPoints": "never",
|
|
"pointSize": 8,
|
|
"gradientMode": "none",
|
|
"spanNulls": false,
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"lineInterpolation": "smooth"
|
|
},
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"min": 0
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"description": "A non-zero waiting count means the pool is saturated: requests queue before they even reach PostgreSQL."
|
|
},
|
|
{
|
|
"id": 18,
|
|
"type": "row",
|
|
"title": "Logs",
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 40
|
|
},
|
|
"panels": []
|
|
},
|
|
{
|
|
"id": 19,
|
|
"type": "timeseries",
|
|
"title": "Log volume by level",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 41
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "sum by (detected_level) (count_over_time({namespace=\"keycloak\",container=\"keycloak\"}[$__auto]))",
|
|
"legendFormat": "{{detected_level}}",
|
|
"queryType": "range"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"min": 0,
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "bars",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 80,
|
|
"showPoints": "never",
|
|
"barAlignment": 0,
|
|
"stacking": {
|
|
"mode": "normal",
|
|
"group": "A"
|
|
},
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto"
|
|
}
|
|
},
|
|
"overrides": [
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "critical"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "dark-red"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "error"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "red"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "warn"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "orange"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "info"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "green"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "debug"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "blue"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "unknown"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "text"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"sum"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"description": "`detected_level` is inferred by Loki at ingestion, so one query works across every log format."
|
|
},
|
|
{
|
|
"id": 20,
|
|
"type": "timeseries",
|
|
"title": "Errors by container (whole namespace)",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 41
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "sum by (container) (count_over_time({namespace=\"keycloak\"} | detected_level =~ \"error|critical\" [$__auto]))",
|
|
"legendFormat": "{{container}}",
|
|
"queryType": "range"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"min": 0,
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "bars",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 80,
|
|
"showPoints": "never",
|
|
"barAlignment": 0,
|
|
"stacking": {
|
|
"mode": "normal",
|
|
"group": "A"
|
|
},
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto"
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"sum"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"description": "Namespace-wide so the datastore's failures show up next to the application's."
|
|
},
|
|
{
|
|
"id": 21,
|
|
"type": "logs",
|
|
"title": "Recent errors and warnings",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"gridPos": {
|
|
"h": 11,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 49
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "{namespace=\"keycloak\"} | detected_level =~ \"error|critical|warn\"",
|
|
"queryType": "range"
|
|
}
|
|
],
|
|
"options": {
|
|
"showTime": true,
|
|
"showLabels": false,
|
|
"showCommonLabels": false,
|
|
"wrapLogMessage": true,
|
|
"prettifyLogMessage": false,
|
|
"sortOrder": "Descending",
|
|
"dedupStrategy": "none",
|
|
"enableLogDetails": true
|
|
},
|
|
"description": "Click a line to expand it. Where the line carries a trace_id, the Loki datasource's derived field opens the matching trace in Tempo."
|
|
},
|
|
{
|
|
"id": 22,
|
|
"type": "row",
|
|
"title": "Container resources",
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 60
|
|
},
|
|
"panels": []
|
|
},
|
|
{
|
|
"id": 23,
|
|
"type": "timeseries",
|
|
"title": "CPU per container",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 61
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum by (pod, container) (rate(container_cpu_usage_seconds_total{namespace=\"keycloak\",pod=~\"keycloak-.*\",container!=\"\"}[$__rate_interval]))",
|
|
"legendFormat": "{{pod}}/{{container}}"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 2,
|
|
"fillOpacity": 8,
|
|
"showPoints": "never",
|
|
"pointSize": 8,
|
|
"gradientMode": "none",
|
|
"spanNulls": false,
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"lineInterpolation": "smooth"
|
|
},
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"min": 0,
|
|
"decimals": 3
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"description": "Cores consumed."
|
|
},
|
|
{
|
|
"id": 24,
|
|
"type": "timeseries",
|
|
"title": "Memory per container",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 61
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum by (pod, container) (container_memory_working_set_bytes{namespace=\"keycloak\",pod=~\"keycloak-.*\",container!=\"\"})",
|
|
"legendFormat": "{{pod}}/{{container}}"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "bytes",
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 2,
|
|
"fillOpacity": 8,
|
|
"showPoints": "never",
|
|
"pointSize": 8,
|
|
"gradientMode": "none",
|
|
"spanNulls": false,
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"lineInterpolation": "smooth"
|
|
},
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"min": 0
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"description": "Working set: what the kernel cannot reclaim under pressure. This is the figure the OOM killer compares against the limit."
|
|
},
|
|
{
|
|
"id": 25,
|
|
"type": "timeseries",
|
|
"title": "Restarts over the window",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 61
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum by (pod) (changes(container_start_time_seconds{namespace=\"keycloak\",pod=~\"keycloak-.*\",container!=\"\"}[$__range]))",
|
|
"legendFormat": "{{pod}}"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 2,
|
|
"fillOpacity": 0,
|
|
"showPoints": "never",
|
|
"pointSize": 8,
|
|
"gradientMode": "none",
|
|
"spanNulls": false,
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"lineInterpolation": "smooth"
|
|
},
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"min": 0,
|
|
"decimals": 0
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"description": "Container start count over the displayed window. Stays at 0 in nominal operation."
|
|
}
|
|
]
|
|
}
|