diff --git a/README.md b/README.md index 1dc9800..17b6bb6 100644 --- a/README.md +++ b/README.md @@ -191,6 +191,14 @@ changes in `container_start_time_seconds`. | `librechat` | OTel SDK → `tempo:4318` | built-in `/metrics`, Bearer token | stdout | `obs-librechat` | | `classic-models-api` | `opentelemetry-instrument` → `tempo:4318` | django-prometheus `/metrics` | JSON + `otelTraceID` | `obs-classic-models-api` | +The six Kong gateways share a single dashboard, `obs-kong-gateways`, with a +`gateway` selector at the top (one, several, or all). Its variable resolves a +subtlety of this setup: the metrics label is the full gateway name +(`catalog-dev`) while the audit-log label is the short domain (`catalog`). The +variable holds the short form, PromQL re-adds the `-dev` suffix, and LogQL uses +it as-is — so one control drives both signals. The capture group in its regex +also drops the unrelated `kong` gateway, which carries no `-dev` suffix. + Their manifests live in the `infra-keycloak`, `infra-librechat` and `classic-models-api` repositories; this repository only holds the platform. diff --git a/k8s/grafana/dashboards/kustomization.yaml b/k8s/grafana/dashboards/kustomization.yaml index 2615de7..ae32b9b 100644 --- a/k8s/grafana/dashboards/kustomization.yaml +++ b/k8s/grafana/dashboards/kustomization.yaml @@ -19,6 +19,7 @@ configMapGenerator: # >>> dashboards (generated by `make dashboards-sync`) >>> - obs-classic-models-api.json - obs-keycloak.json + - obs-kong-gateways.json - obs-librechat.json - obs-stack-health.json # <<< dashboards <<< diff --git a/k8s/grafana/dashboards/obs-kong-gateways.json b/k8s/grafana/dashboards/obs-kong-gateways.json new file mode 100644 index 0000000..00be19f --- /dev/null +++ b/k8s/grafana/dashboards/obs-kong-gateways.json @@ -0,0 +1,1447 @@ +{ + "uid": "obs-kong-gateways", + "title": "Kong Gateways — Overview", + "description": "Traffic, latency, audit logs and health for the Kong gateways. Pick one or several gateways with the selector at the top.", + "tags": [ + "observability", + "kong", + "gateway", + "metrics", + "logs" + ], + "editable": false, + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "time": { + "from": "now-6h", + "to": "now" + }, + "timezone": "browser", + "templating": { + "list": [ + { + "name": "gateway", + "label": "Gateway", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "query": { + "qryType": 1, + "query": "label_values(kong_http_requests_total, gateway)", + "refId": "gateway-variable" + }, + "regex": "/(.*)-dev/", + "refresh": 2, + "sort": 1, + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + } + } + ] + }, + "panels": [ + { + "id": 1, + "type": "row", + "title": "Overview", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "panels": [] + }, + { + "id": 2, + "type": "stat", + "title": "Requests / s", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(kong_http_requests_total{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval]))", + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "mappings": [], + "decimals": 2, + "color": { + "mode": "fixed", + "fixedColor": "blue" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto", + "justifyMode": "auto" + } + }, + { + "id": 3, + "type": "stat", + "title": "5xx rate", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "(sum(rate(kong_http_requests_total{gateway=~\"(${gateway:pipe})-dev\",code=~\"5..\"}[$__rate_interval])) or vector(0)) / clamp_min(sum(rate(kong_http_requests_total{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval])), 0.001)", + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "mappings": [], + "decimals": 2, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.01 + }, + { + "color": "red", + "value": 0.05 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto", + "justifyMode": "auto" + }, + "description": "5xx only: 4xx are largely expected here, the gateways reject unauthenticated calls. `or vector(0)` makes the panel read 0% rather than \"No data\" when nothing fails." + }, + { + "id": 4, + "type": "stat", + "title": "Request latency p95", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(kong_request_latency_ms_bucket{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval])))", + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "mappings": [], + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 500 + }, + { + "color": "red", + "value": 2000 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto", + "justifyMode": "auto" + }, + "description": "Total time the client waited: Kong's own processing plus the upstream." + }, + { + "id": 5, + "type": "stat", + "title": "Upstream latency p95", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(kong_upstream_latency_ms_bucket{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval])))", + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "mappings": [], + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 500 + }, + { + "color": "red", + "value": 2000 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto", + "justifyMode": "auto" + }, + "description": "Time spent waiting on the backend alone. Compare with the panel to its left: the gap is what Kong itself costs." + }, + { + "id": 6, + "type": "row", + "title": "Traffic", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "panels": [] + }, + { + "id": 7, + "type": "timeseries", + "title": "Requests by status class", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (class) (label_replace(rate(kong_http_requests_total{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval]), \"class\", \"$1xx\", \"code\", \"(.)..\"))", + "legendFormat": "{{class}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 70, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "linear", + "stacking": { + "mode": "normal", + "group": "A" + } + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "orange" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + } + ] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "Stacked: total height is throughput, status colours isolate the failing share." + }, + { + "id": 8, + "type": "timeseries", + "title": "Request latency", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 7 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.50, sum by (le) (rate(kong_request_latency_ms_bucket{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval])))", + "legendFormat": "p50" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(kong_request_latency_ms_bucket{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval])))", + "legendFormat": "p95" + }, + { + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(kong_request_latency_ms_bucket{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval])))", + "legendFormat": "p99" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "light-blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p95" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "dark-blue" + } + } + ] + } + ] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "One hue light to dark: p50/p95/p99 are ordered magnitudes of the same measure, not distinct entities." + }, + { + "id": 9, + "type": "timeseries", + "title": "Time spent inside Kong", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 15 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.50, sum by (le) (rate(kong_kong_latency_ms_bucket{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval])))", + "legendFormat": "p50" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(kong_kong_latency_ms_bucket{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval])))", + "legendFormat": "p95" + }, + { + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(kong_kong_latency_ms_bucket{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval])))", + "legendFormat": "p99" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "light-blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p95" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "dark-blue" + } + } + ] + } + ] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "Plugin execution, routing and auth. Rising here without upstream latency rising means the gateway itself is the bottleneck." + }, + { + "id": 10, + "type": "timeseries", + "title": "Upstream latency", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 15 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.50, sum by (le) (rate(kong_upstream_latency_ms_bucket{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval])))", + "legendFormat": "p50" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(kong_upstream_latency_ms_bucket{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval])))", + "legendFormat": "p95" + }, + { + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(kong_upstream_latency_ms_bucket{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval])))", + "legendFormat": "p99" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "light-blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p95" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "dark-blue" + } + } + ] + } + ] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "The backend's own response time, as measured by Kong." + }, + { + "id": 11, + "type": "bargauge", + "title": "Busiest routes", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 23 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "topk(10, sum by (route) (rate(kong_http_requests_total{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval])))", + "legendFormat": "{{route}}", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "min": 0, + "mappings": [], + "color": { + "mode": "continuous-BlPu" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "showUnfilled": true, + "valueMode": "text", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + }, + { + "id": 12, + "type": "timeseries", + "title": "Bandwidth by direction", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 23 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (direction) (rate(kong_bandwidth_bytes{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval]))", + "legendFormat": "{{direction}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "Bps", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 13, + "type": "row", + "title": "Audit logs", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "panels": [] + }, + { + "id": 14, + "type": "timeseries", + "title": "Audit records by status class", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (class) (count_over_time({job=\"kong-audit\", domain=~\"${gateway:pipe}\"} | json status=\"response.status\" | label_format class=`{{ printf \"%.1sxx\" .status }}` [$__auto]))", + "legendFormat": "{{class}}", + "queryType": "range" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "bars", + "lineWidth": 1, + "fillOpacity": 80, + "showPoints": "never", + "barAlignment": 0, + "stacking": { + "mode": "normal", + "group": "A" + }, + "axisLabel": "", + "axisPlacement": "auto" + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "orange" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + } + ] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "sum" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "The status class is derived from the Kong record with LogQL's json parser, not from a log level: these records carry no severity of their own." + }, + { + "id": 15, + "type": "timeseries", + "title": "Failing requests by service", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (svc) (count_over_time({job=\"kong-audit\", domain=~\"${gateway:pipe}\"} | json status=\"response.status\", svc=\"service.name\" | status =~ \"[45]..\" [$__auto]))", + "legendFormat": "{{svc}}", + "queryType": "range" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "bars", + "lineWidth": 1, + "fillOpacity": 80, + "showPoints": "never", + "barAlignment": 0, + "stacking": { + "mode": "normal", + "group": "A" + }, + "axisLabel": "", + "axisPlacement": "auto" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "sum" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "4xx and 5xx only, broken down by the Kong service that answered." + }, + { + "id": 16, + "type": "logs", + "title": "Recent audit records", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 40 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{job=\"kong-audit\", domain=~\"${gateway:pipe}\"} | json", + "queryType": "range" + } + ], + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": true, + "sortOrder": "Descending", + "dedupStrategy": "none", + "enableLogDetails": true + }, + "description": "Full Kong record per request: route, status, latencies, client, and the captured request/response bodies. Where the record carries a trace id, the Loki datasource's derived field opens the matching trace in Tempo." + }, + { + "id": 17, + "type": "row", + "title": "Gateway health", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 52 + }, + "panels": [] + }, + { + "id": 18, + "type": "timeseries", + "title": "nginx connections", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 53 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (state) (kong_nginx_connections_total{gateway=~\"(${gateway:pipe})-dev\"})", + "legendFormat": "{{state}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "Active, reading, writing and waiting connections held by the proxy." + }, + { + "id": 19, + "type": "timeseries", + "title": "Lua shared dict memory", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 53 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (shared_dict) (kong_memory_lua_shared_dict_bytes{gateway=~\"(${gateway:pipe})-dev\"})", + "legendFormat": "{{shared_dict}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "Kong keeps rate-limit counters, caches and cluster state in these. A dict filling up silently degrades the plugin that owns it." + }, + { + "id": 20, + "type": "timeseries", + "title": "Control plane and datastore", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 53 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "min(kong_control_plane_connected{gateway=~\"(${gateway:pipe})-dev\"})", + "legendFormat": "control plane connected" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "min(kong_datastore_reachable{gateway=~\"(${gateway:pipe})-dev\"})", + "legendFormat": "datastore reachable" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "1 is healthy. These are dataplanes attached to Konnect, so losing the control plane means config updates stop arriving while traffic keeps flowing on the last known config." + } + ] +}