{ "uid": "obs-keycloak", "title": "Keycloak — Overview", "description": "HTTP traffic, JVM runtime, connection pool and logs for Keycloak. Latencies come from Tempo span metrics: the Micrometer timer exposes no buckets.", "tags": [ "observability", "keycloak", "application", "metrics", "logs" ], "editable": false, "schemaVersion": 39, "version": 1, "refresh": "30s", "time": { "from": "now-6h", "to": "now" }, "timezone": "browser", "templating": { "list": [] }, "panels": [ { "id": 1, "type": "row", "title": "Overview", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "panels": [] }, { "id": 2, "type": "stat", "title": "Requests / s", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 5, "w": 6, "x": 0, "y": 1 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(http_server_requests_seconds_count{namespace=\"keycloak\"}[$__rate_interval]))", "instant": false } ], "fieldConfig": { "defaults": { "unit": "reqps", "mappings": [], "decimals": 2, "color": { "mode": "fixed", "fixedColor": "blue" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "blue", "value": null } ] } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "value", "graphMode": "area", "textMode": "auto", "justifyMode": "auto" } }, { "id": 3, "type": "stat", "title": "Error rate", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 5, "w": 6, "x": 6, "y": 1 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "(sum(rate(http_server_requests_seconds_count{namespace=\"keycloak\",outcome=~\"CLIENT_ERROR|SERVER_ERROR\"}[$__rate_interval])) or vector(0)) / clamp_min(sum(rate(http_server_requests_seconds_count{namespace=\"keycloak\"}[$__rate_interval])), 0.001)", "instant": false } ], "fieldConfig": { "defaults": { "unit": "percentunit", "mappings": [], "decimals": 2, "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 0.01 }, { "color": "red", "value": 0.05 } ] } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "value", "graphMode": "area", "textMode": "auto", "justifyMode": "auto" }, "description": "Share of 4xx and 5xx responses. `or vector(0)` is required: with no errors at all the numerator is an empty vector, and empty/x stays empty -- the panel would read \"No data\" instead of 0%. clamp_min covers the zero-traffic case." }, { "id": 4, "type": "stat", "title": "Latency p95", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 5, "w": 6, "x": 12, "y": 1 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "histogram_quantile(0.95, sum by (le) (rate(traces_spanmetrics_latency_bucket{service=\"keycloak\",span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval])))", "instant": false } ], "fieldConfig": { "defaults": { "unit": "s", "mappings": [], "decimals": 3, "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 0.5 }, { "color": "red", "value": 2 } ] } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "value", "graphMode": "area", "textMode": "auto", "justifyMode": "auto" }, "description": "Source: span metrics generated by Tempo. Keycloak's Micrometer timer exposes neither buckets nor quantiles, so no percentile is computable from /metrics." }, { "id": 5, "type": "stat", "title": "JVM heap used", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 5, "w": 6, "x": 18, "y": 1 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(jvm_memory_used_bytes{namespace=\"keycloak\",area=\"heap\"})", "instant": false } ], "fieldConfig": { "defaults": { "unit": "bytes", "mappings": [], "color": { "mode": "fixed", "fixedColor": "purple" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "purple", "value": null } ] } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "value", "graphMode": "area", "textMode": "auto", "justifyMode": "auto" } }, { "id": 6, "type": "row", "title": "HTTP traffic", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 6 }, "panels": [] }, { "id": 7, "type": "timeseries", "title": "Requests by outcome", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 7 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (outcome) (rate(http_server_requests_seconds_count{namespace=\"keycloak\"}[$__rate_interval]))", "legendFormat": "{{outcome}}" } ], "fieldConfig": { "defaults": { "unit": "reqps", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 70, "showPoints": "never", "pointSize": 8, "gradientMode": "none", "spanNulls": false, "axisLabel": "", "axisPlacement": "auto", "lineInterpolation": "linear", "stacking": { "mode": "normal", "group": "A" } }, "mappings": [], "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "2xx" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } } ] }, { "matcher": { "id": "byName", "options": "3xx" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } } ] }, { "matcher": { "id": "byName", "options": "4xx" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "orange" } } ] }, { "matcher": { "id": "byName", "options": "5xx" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } ] }, { "matcher": { "id": "byName", "options": "SUCCESS" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } } ] }, { "matcher": { "id": "byName", "options": "REDIRECTION" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } } ] }, { "matcher": { "id": "byName", "options": "CLIENT_ERROR" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "orange" } } ] }, { "matcher": { "id": "byName", "options": "SERVER_ERROR" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } ] } ] }, "options": { "legend": { "showLegend": true, "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "Stacked: total height is throughput, status colours isolate the failing share." }, { "id": 8, "type": "timeseries", "title": "Latency (Tempo span metrics)", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 7 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "histogram_quantile(0.50, sum by (le) (rate(traces_spanmetrics_latency_bucket{service=\"keycloak\",span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval])))", "legendFormat": "p50" }, { "refId": "B", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "histogram_quantile(0.95, sum by (le) (rate(traces_spanmetrics_latency_bucket{service=\"keycloak\",span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval])))", "legendFormat": "p95" }, { "refId": "C", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "histogram_quantile(0.99, sum by (le) (rate(traces_spanmetrics_latency_bucket{service=\"keycloak\",span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval])))", "legendFormat": "p99" } ], "fieldConfig": { "defaults": { "unit": "s", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "pointSize": 8, "gradientMode": "none", "spanNulls": false, "axisLabel": "", "axisPlacement": "auto", "lineInterpolation": "smooth" }, "mappings": [], "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "p50" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "light-blue" } } ] }, { "matcher": { "id": "byName", "options": "p95" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } } ] }, { "matcher": { "id": "byName", "options": "p99" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "dark-blue" } } ] } ] }, "options": { "legend": { "showLegend": true, "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "One hue from light to dark: p50/p95/p99 are ordered magnitudes of the same measure, not distinct entities." }, { "id": 9, "type": "bargauge", "title": "Busiest endpoints", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 15 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "topk(10, sum by (uri) (rate(http_server_requests_seconds_count{namespace=\"keycloak\"}[$__rate_interval])))", "legendFormat": "{{uri}}", "instant": true } ], "fieldConfig": { "defaults": { "unit": "reqps", "min": 0, "mappings": [], "color": { "mode": "continuous-BlPu" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "blue", "value": null } ] } }, "overrides": [] }, "options": { "displayMode": "gradient", "orientation": "horizontal", "showUnfilled": true, "valueMode": "text", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } } }, { "id": 10, "type": "timeseries", "title": "Mean latency per endpoint", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 15 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "topk(10, sum by (uri) (rate(http_server_requests_seconds_sum{namespace=\"keycloak\"}[$__rate_interval])) / clamp_min(sum by (uri) (rate(http_server_requests_seconds_count{namespace=\"keycloak\"}[$__rate_interval])), 0.001))", "legendFormat": "{{uri}}" } ], "fieldConfig": { "defaults": { "unit": "s", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "pointSize": 8, "gradientMode": "none", "spanNulls": false, "axisLabel": "", "axisPlacement": "auto", "lineInterpolation": "smooth" }, "mappings": [], "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "A mean, not a percentile: that is all a bucket-less timer allows. A mean hides the tail -- for the tail, see the p99 panel." }, { "id": 11, "type": "row", "title": "JVM runtime", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 }, "panels": [] }, { "id": 12, "type": "timeseries", "title": "JVM memory by area", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 24 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (area) (jvm_memory_used_bytes{namespace=\"keycloak\"})", "legendFormat": "{{area}}" }, { "refId": "B", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(jvm_memory_max_bytes{namespace=\"keycloak\",area=\"heap\"} > 0)", "legendFormat": "max heap" } ], "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 8, "showPoints": "never", "pointSize": 8, "gradientMode": "none", "spanNulls": false, "axisLabel": "", "axisPlacement": "auto", "lineInterpolation": "smooth" }, "mappings": [], "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 13, "type": "timeseries", "title": "GC pauses", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 24 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(jvm_gc_pause_seconds_sum{namespace=\"keycloak\"}[$__rate_interval])) / clamp_min(sum(rate(jvm_gc_pause_seconds_count{namespace=\"keycloak\"}[$__rate_interval])), 0.001)", "legendFormat": "mean duration" } ], "fieldConfig": { "defaults": { "unit": "s", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 8, "showPoints": "never", "pointSize": 8, "gradientMode": "none", "spanNulls": false, "axisLabel": "", "axisPlacement": "auto", "lineInterpolation": "smooth" }, "mappings": [], "color": { "mode": "palette-classic" }, "min": 0, "decimals": 4 }, "overrides": [] }, "options": { "legend": { "showLegend": false, "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "Mean GC pause duration. Drift here usually precedes a rise in p99 latency." }, { "id": 14, "type": "timeseries", "title": "JVM threads", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 24 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(jvm_threads_live_threads{namespace=\"keycloak\"})", "legendFormat": "live" }, { "refId": "B", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(jvm_threads_daemon_threads{namespace=\"keycloak\"})", "legendFormat": "daemon" }, { "refId": "C", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(jvm_threads_peak_threads{namespace=\"keycloak\"})", "legendFormat": "peak" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "pointSize": 8, "gradientMode": "none", "spanNulls": false, "axisLabel": "", "axisPlacement": "auto", "lineInterpolation": "smooth" }, "mappings": [], "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 15, "type": "row", "title": "PostgreSQL connection pool (Agroal)", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 }, "panels": [] }, { "id": 16, "type": "timeseries", "title": "Pool connections", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 33 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (datasource) (agroal_active_count{namespace=\"keycloak\"})", "legendFormat": "active" }, { "refId": "B", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (datasource) (agroal_available_count{namespace=\"keycloak\"})", "legendFormat": "available" }, { "refId": "C", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (datasource) (agroal_max_used_count{namespace=\"keycloak\"})", "legendFormat": "peak used" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "pointSize": 8, "gradientMode": "none", "spanNulls": false, "axisLabel": "", "axisPlacement": "auto", "lineInterpolation": "smooth" }, "mappings": [], "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 17, "type": "timeseries", "title": "Waiting for a connection", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 33 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(agroal_awaiting_count{namespace=\"keycloak\"})", "legendFormat": "requests waiting" }, { "refId": "B", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(agroal_blocking_time_average_milliseconds{namespace=\"keycloak\"})", "legendFormat": "mean blocking time (ms)" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "pointSize": 8, "gradientMode": "none", "spanNulls": false, "axisLabel": "", "axisPlacement": "auto", "lineInterpolation": "smooth" }, "mappings": [], "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "A non-zero waiting count means the pool is saturated: requests queue before they even reach PostgreSQL." }, { "id": 18, "type": "row", "title": "Logs", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 40 }, "panels": [] }, { "id": 19, "type": "timeseries", "title": "Log volume by level", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 41 }, "targets": [ { "refId": "A", "datasource": { "type": "loki", "uid": "loki" }, "expr": "sum by (detected_level) (count_over_time({namespace=\"keycloak\",container=\"keycloak\"}[$__auto]))", "legendFormat": "{{detected_level}}", "queryType": "range" } ], "fieldConfig": { "defaults": { "unit": "short", "min": 0, "mappings": [], "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "bars", "lineWidth": 1, "fillOpacity": 80, "showPoints": "never", "barAlignment": 0, "stacking": { "mode": "normal", "group": "A" }, "axisLabel": "", "axisPlacement": "auto" } }, "overrides": [ { "matcher": { "id": "byName", "options": "critical" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "dark-red" } } ] }, { "matcher": { "id": "byName", "options": "error" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } ] }, { "matcher": { "id": "byName", "options": "warn" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "orange" } } ] }, { "matcher": { "id": "byName", "options": "info" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } } ] }, { "matcher": { "id": "byName", "options": "debug" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } } ] }, { "matcher": { "id": "byName", "options": "unknown" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "text" } } ] } ] }, "options": { "legend": { "showLegend": true, "displayMode": "table", "placement": "bottom", "calcs": [ "sum" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "`detected_level` is inferred by Loki at ingestion, so one query works across every log format." }, { "id": 20, "type": "timeseries", "title": "Errors by container (whole namespace)", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 41 }, "targets": [ { "refId": "A", "datasource": { "type": "loki", "uid": "loki" }, "expr": "sum by (container) (count_over_time({namespace=\"keycloak\"} | detected_level =~ \"error|critical\" [$__auto]))", "legendFormat": "{{container}}", "queryType": "range" } ], "fieldConfig": { "defaults": { "unit": "short", "min": 0, "mappings": [], "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "bars", "lineWidth": 1, "fillOpacity": 80, "showPoints": "never", "barAlignment": 0, "stacking": { "mode": "normal", "group": "A" }, "axisLabel": "", "axisPlacement": "auto" } }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "table", "placement": "bottom", "calcs": [ "sum" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "Namespace-wide so the datastore's failures show up next to the application's." }, { "id": 26, "type": "table", "title": "Errors and warnings", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 11, "w": 24, "x": 0, "y": 49 }, "targets": [ { "refId": "A", "datasource": { "type": "loki", "uid": "loki" }, "expr": "{namespace=\"keycloak\", container=\"keycloak\"} | json | detected_level =~ \"error|critical|warn\"", "queryType": "range", "maxLines": 500 } ], "transformations": [ { "id": "extractFields", "options": { "source": "labels", "replace": false } }, { "id": "filterFieldsByName", "options": { "include": { "pattern": "^(timestamp|body|level|loggerName|message|threadName)$" } } }, { "id": "organize", "options": { "excludeByName": {}, "indexByName": { "Time": 0, "level": 1, "loggerName": 2, "message": 3, "threadName": 4, "body": 5 }, "renameByName": { "level": "Level", "loggerName": "Logger", "message": "Message", "threadName": "Thread", "body": "Raw record", "timestamp": "Time" } } } ], "fieldConfig": { "defaults": { "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": true, "filterable": true }, "mappings": [] }, "overrides": [ { "matcher": { "id": "byName", "options": "Level" }, "properties": [ { "id": "custom.width", "value": 90 }, { "id": "mappings", "value": [ { "type": "regex", "options": { "pattern": "(?i)^(fatal|critical)$", "result": { "color": "dark-red", "index": 0 } } }, { "type": "regex", "options": { "pattern": "(?i)^error$", "result": { "color": "red", "index": 1 } } }, { "type": "regex", "options": { "pattern": "(?i)^warn(ing)?$", "result": { "color": "orange", "index": 2 } } }, { "type": "regex", "options": { "pattern": "(?i)^info$", "result": { "color": "green", "index": 3 } } }, { "type": "regex", "options": { "pattern": "(?i)^debug$", "result": { "color": "blue", "index": 4 } } } ] }, { "id": "custom.cellOptions", "value": { "type": "color-text" } } ] }, { "matcher": { "id": "byName", "options": "Logger" }, "properties": [ { "id": "custom.width", "value": 280 } ] }, { "matcher": { "id": "byName", "options": "Thread" }, "properties": [ { "id": "custom.width", "value": 160 } ] }, { "matcher": { "id": "byName", "options": "Raw record" }, "properties": [ { "id": "custom.width", "value": 260 } ] } ] }, "options": { "showHeader": true, "cellHeight": "sm", "footer": { "show": false, "reducer": [ "sum" ], "countRows": false, "fields": "" }, "sortBy": [ { "displayName": "Time", "desc": true } ] }, "description": "Keycloak logs JSON, so the fields come straight from `| json`. Scoped to the Keycloak container: PostgreSQL in the same namespace is not JSON and would parse to empty columns." }, { "id": 27, "type": "row", "title": "Raw log lines", "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 60 }, "panels": [ { "id": 21, "type": "logs", "title": "Raw log lines", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 11, "w": 24, "x": 0, "y": 61 }, "targets": [ { "refId": "A", "datasource": { "type": "loki", "uid": "loki" }, "expr": "{namespace=\"keycloak\"} | detected_level =~ \"error|critical|warn\"", "queryType": "range" } ], "options": { "showTime": true, "showLabels": false, "showCommonLabels": false, "wrapLogMessage": true, "prettifyLogMessage": false, "sortOrder": "Descending", "dedupStrategy": "none", "enableLogDetails": true }, "description": "Kept alongside the table: the logs panel renders ANSI colour, expands a record's detail inline, tails live, and turns a trace id into a link to Tempo — none of which a table can do. Collapsed so it stays out of the way until needed." } ] }, { "id": 22, "type": "row", "title": "Container resources", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 61 }, "panels": [] }, { "id": 23, "type": "timeseries", "title": "CPU per container", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 62 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (pod, container) (rate(container_cpu_usage_seconds_total{namespace=\"keycloak\",pod=~\"keycloak-.*\",container!=\"\"}[$__rate_interval]))", "legendFormat": "{{pod}}/{{container}}" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 8, "showPoints": "never", "pointSize": 8, "gradientMode": "none", "spanNulls": false, "axisLabel": "", "axisPlacement": "auto", "lineInterpolation": "smooth" }, "mappings": [], "color": { "mode": "palette-classic" }, "min": 0, "decimals": 3 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "Cores consumed." }, { "id": 24, "type": "timeseries", "title": "Memory per container", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 62 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (pod, container) (container_memory_working_set_bytes{namespace=\"keycloak\",pod=~\"keycloak-.*\",container!=\"\"})", "legendFormat": "{{pod}}/{{container}}" } ], "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 8, "showPoints": "never", "pointSize": 8, "gradientMode": "none", "spanNulls": false, "axisLabel": "", "axisPlacement": "auto", "lineInterpolation": "smooth" }, "mappings": [], "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "Working set: what the kernel cannot reclaim under pressure. This is the figure the OOM killer compares against the limit." }, { "id": 25, "type": "timeseries", "title": "Restarts over the window", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 62 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (pod) (changes(container_start_time_seconds{namespace=\"keycloak\",pod=~\"keycloak-.*\",container!=\"\"}[$__range]))", "legendFormat": "{{pod}}" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "pointSize": 8, "gradientMode": "none", "spanNulls": false, "axisLabel": "", "axisPlacement": "auto", "lineInterpolation": "smooth" }, "mappings": [], "color": { "mode": "palette-classic" }, "min": 0, "decimals": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "Container start count over the displayed window. Stays at 0 in nominal operation." } ] }