From 0ba38bc8f742772f12be1762eaa44a50b99289bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Tailland?= Date: Tue, 25 Aug 2026 18:02:07 +0200 Subject: [PATCH] Add a Kong gateways dashboard One dashboard for all six gateways, with a selector accepting one, several or all of them. The selector has to bridge a mismatch: metrics carry the full gateway name (catalog-dev) while audit logs carry the short domain (catalog), because the former comes from the scrape job's label and the latter from the URI path each gateway posts its http-log to. The variable holds the short form, PromQL re-adds the -dev suffix and LogQL uses it as-is, so a single control drives both signals. The capture group in the variable's regex also excludes the unrelated `kong` gateway, which has no -dev suffix. Latency is split three ways rather than shown as one number: total request latency, time spent inside Kong, and upstream latency. The gap between the first and the last is what the gateway itself costs, which is the question worth asking of a proxy. Status classes come from label_replace on the `code` label for metrics and from LogQL's json parser for the audit records, since those carry no severity of their own. Co-Authored-By: Claude Opus 5 --- README.md | 8 + k8s/grafana/dashboards/kustomization.yaml | 1 + k8s/grafana/dashboards/obs-kong-gateways.json | 1447 +++++++++++++++++ 3 files changed, 1456 insertions(+) create mode 100644 k8s/grafana/dashboards/obs-kong-gateways.json diff --git a/README.md b/README.md index 1dc9800..17b6bb6 100644 --- a/README.md +++ b/README.md @@ -191,6 +191,14 @@ changes in `container_start_time_seconds`. | `librechat` | OTel SDK → `tempo:4318` | built-in `/metrics`, Bearer token | stdout | `obs-librechat` | | `classic-models-api` | `opentelemetry-instrument` → `tempo:4318` | django-prometheus `/metrics` | JSON + `otelTraceID` | `obs-classic-models-api` | +The six Kong gateways share a single dashboard, `obs-kong-gateways`, with a +`gateway` selector at the top (one, several, or all). Its variable resolves a +subtlety of this setup: the metrics label is the full gateway name +(`catalog-dev`) while the audit-log label is the short domain (`catalog`). The +variable holds the short form, PromQL re-adds the `-dev` suffix, and LogQL uses +it as-is — so one control drives both signals. The capture group in its regex +also drops the unrelated `kong` gateway, which carries no `-dev` suffix. + Their manifests live in the `infra-keycloak`, `infra-librechat` and `classic-models-api` repositories; this repository only holds the platform. diff --git a/k8s/grafana/dashboards/kustomization.yaml b/k8s/grafana/dashboards/kustomization.yaml index 2615de7..ae32b9b 100644 --- a/k8s/grafana/dashboards/kustomization.yaml +++ b/k8s/grafana/dashboards/kustomization.yaml @@ -19,6 +19,7 @@ configMapGenerator: # >>> dashboards (generated by `make dashboards-sync`) >>> - obs-classic-models-api.json - obs-keycloak.json + - obs-kong-gateways.json - obs-librechat.json - obs-stack-health.json # <<< dashboards <<< diff --git a/k8s/grafana/dashboards/obs-kong-gateways.json b/k8s/grafana/dashboards/obs-kong-gateways.json new file mode 100644 index 0000000..00be19f --- /dev/null +++ b/k8s/grafana/dashboards/obs-kong-gateways.json @@ -0,0 +1,1447 @@ +{ + "uid": "obs-kong-gateways", + "title": "Kong Gateways — Overview", + "description": "Traffic, latency, audit logs and health for the Kong gateways. Pick one or several gateways with the selector at the top.", + "tags": [ + "observability", + "kong", + "gateway", + "metrics", + "logs" + ], + "editable": false, + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "time": { + "from": "now-6h", + "to": "now" + }, + "timezone": "browser", + "templating": { + "list": [ + { + "name": "gateway", + "label": "Gateway", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "query": { + "qryType": 1, + "query": "label_values(kong_http_requests_total, gateway)", + "refId": "gateway-variable" + }, + "regex": "/(.*)-dev/", + "refresh": 2, + "sort": 1, + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + } + } + ] + }, + "panels": [ + { + "id": 1, + "type": "row", + "title": "Overview", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "panels": [] + }, + { + "id": 2, + "type": "stat", + "title": "Requests / s", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(kong_http_requests_total{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval]))", + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "mappings": [], + "decimals": 2, + "color": { + "mode": "fixed", + "fixedColor": "blue" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto", + "justifyMode": "auto" + } + }, + { + "id": 3, + "type": "stat", + "title": "5xx rate", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "(sum(rate(kong_http_requests_total{gateway=~\"(${gateway:pipe})-dev\",code=~\"5..\"}[$__rate_interval])) or vector(0)) / clamp_min(sum(rate(kong_http_requests_total{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval])), 0.001)", + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "mappings": [], + "decimals": 2, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.01 + }, + { + "color": "red", + "value": 0.05 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto", + "justifyMode": "auto" + }, + "description": "5xx only: 4xx are largely expected here, the gateways reject unauthenticated calls. `or vector(0)` makes the panel read 0% rather than \"No data\" when nothing fails." + }, + { + "id": 4, + "type": "stat", + "title": "Request latency p95", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(kong_request_latency_ms_bucket{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval])))", + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "mappings": [], + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 500 + }, + { + "color": "red", + "value": 2000 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto", + "justifyMode": "auto" + }, + "description": "Total time the client waited: Kong's own processing plus the upstream." + }, + { + "id": 5, + "type": "stat", + "title": "Upstream latency p95", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(kong_upstream_latency_ms_bucket{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval])))", + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "mappings": [], + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 500 + }, + { + "color": "red", + "value": 2000 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto", + "justifyMode": "auto" + }, + "description": "Time spent waiting on the backend alone. Compare with the panel to its left: the gap is what Kong itself costs." + }, + { + "id": 6, + "type": "row", + "title": "Traffic", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "panels": [] + }, + { + "id": 7, + "type": "timeseries", + "title": "Requests by status class", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (class) (label_replace(rate(kong_http_requests_total{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval]), \"class\", \"$1xx\", \"code\", \"(.)..\"))", + "legendFormat": "{{class}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 70, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "linear", + "stacking": { + "mode": "normal", + "group": "A" + } + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "orange" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + } + ] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "Stacked: total height is throughput, status colours isolate the failing share." + }, + { + "id": 8, + "type": "timeseries", + "title": "Request latency", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 7 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.50, sum by (le) (rate(kong_request_latency_ms_bucket{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval])))", + "legendFormat": "p50" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(kong_request_latency_ms_bucket{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval])))", + "legendFormat": "p95" + }, + { + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(kong_request_latency_ms_bucket{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval])))", + "legendFormat": "p99" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "light-blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p95" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "dark-blue" + } + } + ] + } + ] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "One hue light to dark: p50/p95/p99 are ordered magnitudes of the same measure, not distinct entities." + }, + { + "id": 9, + "type": "timeseries", + "title": "Time spent inside Kong", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 15 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.50, sum by (le) (rate(kong_kong_latency_ms_bucket{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval])))", + "legendFormat": "p50" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(kong_kong_latency_ms_bucket{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval])))", + "legendFormat": "p95" + }, + { + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(kong_kong_latency_ms_bucket{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval])))", + "legendFormat": "p99" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "light-blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p95" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "dark-blue" + } + } + ] + } + ] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "Plugin execution, routing and auth. Rising here without upstream latency rising means the gateway itself is the bottleneck." + }, + { + "id": 10, + "type": "timeseries", + "title": "Upstream latency", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 15 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.50, sum by (le) (rate(kong_upstream_latency_ms_bucket{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval])))", + "legendFormat": "p50" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(kong_upstream_latency_ms_bucket{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval])))", + "legendFormat": "p95" + }, + { + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(kong_upstream_latency_ms_bucket{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval])))", + "legendFormat": "p99" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "light-blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p95" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "dark-blue" + } + } + ] + } + ] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "The backend's own response time, as measured by Kong." + }, + { + "id": 11, + "type": "bargauge", + "title": "Busiest routes", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 23 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "topk(10, sum by (route) (rate(kong_http_requests_total{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval])))", + "legendFormat": "{{route}}", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "min": 0, + "mappings": [], + "color": { + "mode": "continuous-BlPu" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "showUnfilled": true, + "valueMode": "text", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + }, + { + "id": 12, + "type": "timeseries", + "title": "Bandwidth by direction", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 23 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (direction) (rate(kong_bandwidth_bytes{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval]))", + "legendFormat": "{{direction}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "Bps", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 13, + "type": "row", + "title": "Audit logs", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "panels": [] + }, + { + "id": 14, + "type": "timeseries", + "title": "Audit records by status class", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (class) (count_over_time({job=\"kong-audit\", domain=~\"${gateway:pipe}\"} | json status=\"response.status\" | label_format class=`{{ printf \"%.1sxx\" .status }}` [$__auto]))", + "legendFormat": "{{class}}", + "queryType": "range" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "bars", + "lineWidth": 1, + "fillOpacity": 80, + "showPoints": "never", + "barAlignment": 0, + "stacking": { + "mode": "normal", + "group": "A" + }, + "axisLabel": "", + "axisPlacement": "auto" + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "orange" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + } + ] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "sum" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "The status class is derived from the Kong record with LogQL's json parser, not from a log level: these records carry no severity of their own." + }, + { + "id": 15, + "type": "timeseries", + "title": "Failing requests by service", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (svc) (count_over_time({job=\"kong-audit\", domain=~\"${gateway:pipe}\"} | json status=\"response.status\", svc=\"service.name\" | status =~ \"[45]..\" [$__auto]))", + "legendFormat": "{{svc}}", + "queryType": "range" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "bars", + "lineWidth": 1, + "fillOpacity": 80, + "showPoints": "never", + "barAlignment": 0, + "stacking": { + "mode": "normal", + "group": "A" + }, + "axisLabel": "", + "axisPlacement": "auto" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "sum" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "4xx and 5xx only, broken down by the Kong service that answered." + }, + { + "id": 16, + "type": "logs", + "title": "Recent audit records", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 40 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{job=\"kong-audit\", domain=~\"${gateway:pipe}\"} | json", + "queryType": "range" + } + ], + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": true, + "sortOrder": "Descending", + "dedupStrategy": "none", + "enableLogDetails": true + }, + "description": "Full Kong record per request: route, status, latencies, client, and the captured request/response bodies. Where the record carries a trace id, the Loki datasource's derived field opens the matching trace in Tempo." + }, + { + "id": 17, + "type": "row", + "title": "Gateway health", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 52 + }, + "panels": [] + }, + { + "id": 18, + "type": "timeseries", + "title": "nginx connections", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 53 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (state) (kong_nginx_connections_total{gateway=~\"(${gateway:pipe})-dev\"})", + "legendFormat": "{{state}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "Active, reading, writing and waiting connections held by the proxy." + }, + { + "id": 19, + "type": "timeseries", + "title": "Lua shared dict memory", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 53 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (shared_dict) (kong_memory_lua_shared_dict_bytes{gateway=~\"(${gateway:pipe})-dev\"})", + "legendFormat": "{{shared_dict}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "Kong keeps rate-limit counters, caches and cluster state in these. A dict filling up silently degrades the plugin that owns it." + }, + { + "id": 20, + "type": "timeseries", + "title": "Control plane and datastore", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 53 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "min(kong_control_plane_connected{gateway=~\"(${gateway:pipe})-dev\"})", + "legendFormat": "control plane connected" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "min(kong_datastore_reachable{gateway=~\"(${gateway:pipe})-dev\"})", + "legendFormat": "datastore reachable" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "1 is healthy. These are dataplanes attached to Konnect, so losing the control plane means config updates stop arriving while traffic keeps flowing on the last known config." + } + ] +}