Stéphane Tailland
0ba38bc8f7
One dashboard for all six gateways, with a selector accepting one, several or all of them. The selector has to bridge a mismatch: metrics carry the full gateway name (catalog-dev) while audit logs carry the short domain (catalog), because the former comes from the scrape job's label and the latter from the URI path each gateway posts its http-log to. The variable holds the short form, PromQL re-adds the -dev suffix and LogQL uses it as-is, so a single control drives both signals. The capture group in the variable's regex also excludes the unrelated `kong` gateway, which has no -dev suffix. Latency is split three ways rather than shown as one number: total request latency, time spent inside Kong, and upstream latency. The gap between the first and the last is what the gateway itself costs, which is the question worth asking of a proxy. Status classes come from label_replace on the `code` label for metrics and from LogQL's json parser for the audit records, since those carry no severity of their own. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
1448 lines
35 KiB
JSON
1448 lines
35 KiB
JSON
{
|
|
"uid": "obs-kong-gateways",
|
|
"title": "Kong Gateways — Overview",
|
|
"description": "Traffic, latency, audit logs and health for the Kong gateways. Pick one or several gateways with the selector at the top.",
|
|
"tags": [
|
|
"observability",
|
|
"kong",
|
|
"gateway",
|
|
"metrics",
|
|
"logs"
|
|
],
|
|
"editable": false,
|
|
"schemaVersion": 39,
|
|
"version": 1,
|
|
"refresh": "30s",
|
|
"time": {
|
|
"from": "now-6h",
|
|
"to": "now"
|
|
},
|
|
"timezone": "browser",
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"name": "gateway",
|
|
"label": "Gateway",
|
|
"type": "query",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"query": {
|
|
"qryType": 1,
|
|
"query": "label_values(kong_http_requests_total, gateway)",
|
|
"refId": "gateway-variable"
|
|
},
|
|
"regex": "/(.*)-dev/",
|
|
"refresh": 2,
|
|
"sort": 1,
|
|
"multi": true,
|
|
"includeAll": true,
|
|
"allValue": ".*",
|
|
"current": {
|
|
"text": [
|
|
"All"
|
|
],
|
|
"value": [
|
|
"$__all"
|
|
]
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"panels": [
|
|
{
|
|
"id": 1,
|
|
"type": "row",
|
|
"title": "Overview",
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 0
|
|
},
|
|
"panels": []
|
|
},
|
|
{
|
|
"id": 2,
|
|
"type": "stat",
|
|
"title": "Requests / s",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 6,
|
|
"x": 0,
|
|
"y": 1
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum(rate(kong_http_requests_total{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval]))",
|
|
"instant": false
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "reqps",
|
|
"mappings": [],
|
|
"decimals": 2,
|
|
"color": {
|
|
"mode": "fixed",
|
|
"fixedColor": "blue"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "blue",
|
|
"value": null
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"textMode": "auto",
|
|
"justifyMode": "auto"
|
|
}
|
|
},
|
|
{
|
|
"id": 3,
|
|
"type": "stat",
|
|
"title": "5xx rate",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 6,
|
|
"x": 6,
|
|
"y": 1
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "(sum(rate(kong_http_requests_total{gateway=~\"(${gateway:pipe})-dev\",code=~\"5..\"}[$__rate_interval])) or vector(0)) / clamp_min(sum(rate(kong_http_requests_total{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval])), 0.001)",
|
|
"instant": false
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percentunit",
|
|
"mappings": [],
|
|
"decimals": 2,
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "orange",
|
|
"value": 0.01
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 0.05
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"textMode": "auto",
|
|
"justifyMode": "auto"
|
|
},
|
|
"description": "5xx only: 4xx are largely expected here, the gateways reject unauthenticated calls. `or vector(0)` makes the panel read 0% rather than \"No data\" when nothing fails."
|
|
},
|
|
{
|
|
"id": 4,
|
|
"type": "stat",
|
|
"title": "Request latency p95",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 6,
|
|
"x": 12,
|
|
"y": 1
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "histogram_quantile(0.95, sum by (le) (rate(kong_request_latency_ms_bucket{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval])))",
|
|
"instant": false
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "ms",
|
|
"mappings": [],
|
|
"decimals": 0,
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "orange",
|
|
"value": 500
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 2000
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"textMode": "auto",
|
|
"justifyMode": "auto"
|
|
},
|
|
"description": "Total time the client waited: Kong's own processing plus the upstream."
|
|
},
|
|
{
|
|
"id": 5,
|
|
"type": "stat",
|
|
"title": "Upstream latency p95",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 6,
|
|
"x": 18,
|
|
"y": 1
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "histogram_quantile(0.95, sum by (le) (rate(kong_upstream_latency_ms_bucket{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval])))",
|
|
"instant": false
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "ms",
|
|
"mappings": [],
|
|
"decimals": 0,
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "orange",
|
|
"value": 500
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 2000
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"textMode": "auto",
|
|
"justifyMode": "auto"
|
|
},
|
|
"description": "Time spent waiting on the backend alone. Compare with the panel to its left: the gap is what Kong itself costs."
|
|
},
|
|
{
|
|
"id": 6,
|
|
"type": "row",
|
|
"title": "Traffic",
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 6
|
|
},
|
|
"panels": []
|
|
},
|
|
{
|
|
"id": 7,
|
|
"type": "timeseries",
|
|
"title": "Requests by status class",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 7
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum by (class) (label_replace(rate(kong_http_requests_total{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval]), \"class\", \"$1xx\", \"code\", \"(.)..\"))",
|
|
"legendFormat": "{{class}}"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "reqps",
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 70,
|
|
"showPoints": "never",
|
|
"pointSize": 8,
|
|
"gradientMode": "none",
|
|
"spanNulls": false,
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"lineInterpolation": "linear",
|
|
"stacking": {
|
|
"mode": "normal",
|
|
"group": "A"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"min": 0
|
|
},
|
|
"overrides": [
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "2xx"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "green"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "3xx"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "blue"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "4xx"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "orange"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "5xx"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "red"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"description": "Stacked: total height is throughput, status colours isolate the failing share."
|
|
},
|
|
{
|
|
"id": 8,
|
|
"type": "timeseries",
|
|
"title": "Request latency",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 7
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "histogram_quantile(0.50, sum by (le) (rate(kong_request_latency_ms_bucket{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval])))",
|
|
"legendFormat": "p50"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "histogram_quantile(0.95, sum by (le) (rate(kong_request_latency_ms_bucket{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval])))",
|
|
"legendFormat": "p95"
|
|
},
|
|
{
|
|
"refId": "C",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "histogram_quantile(0.99, sum by (le) (rate(kong_request_latency_ms_bucket{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval])))",
|
|
"legendFormat": "p99"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "ms",
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 2,
|
|
"fillOpacity": 0,
|
|
"showPoints": "never",
|
|
"pointSize": 8,
|
|
"gradientMode": "none",
|
|
"spanNulls": false,
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"lineInterpolation": "smooth"
|
|
},
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"min": 0
|
|
},
|
|
"overrides": [
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "p50"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "light-blue"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "p95"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "blue"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "p99"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "dark-blue"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"description": "One hue light to dark: p50/p95/p99 are ordered magnitudes of the same measure, not distinct entities."
|
|
},
|
|
{
|
|
"id": 9,
|
|
"type": "timeseries",
|
|
"title": "Time spent inside Kong",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 15
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "histogram_quantile(0.50, sum by (le) (rate(kong_kong_latency_ms_bucket{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval])))",
|
|
"legendFormat": "p50"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "histogram_quantile(0.95, sum by (le) (rate(kong_kong_latency_ms_bucket{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval])))",
|
|
"legendFormat": "p95"
|
|
},
|
|
{
|
|
"refId": "C",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "histogram_quantile(0.99, sum by (le) (rate(kong_kong_latency_ms_bucket{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval])))",
|
|
"legendFormat": "p99"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "ms",
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 2,
|
|
"fillOpacity": 0,
|
|
"showPoints": "never",
|
|
"pointSize": 8,
|
|
"gradientMode": "none",
|
|
"spanNulls": false,
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"lineInterpolation": "smooth"
|
|
},
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"min": 0
|
|
},
|
|
"overrides": [
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "p50"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "light-blue"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "p95"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "blue"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "p99"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "dark-blue"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"description": "Plugin execution, routing and auth. Rising here without upstream latency rising means the gateway itself is the bottleneck."
|
|
},
|
|
{
|
|
"id": 10,
|
|
"type": "timeseries",
|
|
"title": "Upstream latency",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 15
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "histogram_quantile(0.50, sum by (le) (rate(kong_upstream_latency_ms_bucket{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval])))",
|
|
"legendFormat": "p50"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "histogram_quantile(0.95, sum by (le) (rate(kong_upstream_latency_ms_bucket{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval])))",
|
|
"legendFormat": "p95"
|
|
},
|
|
{
|
|
"refId": "C",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "histogram_quantile(0.99, sum by (le) (rate(kong_upstream_latency_ms_bucket{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval])))",
|
|
"legendFormat": "p99"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "ms",
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 2,
|
|
"fillOpacity": 0,
|
|
"showPoints": "never",
|
|
"pointSize": 8,
|
|
"gradientMode": "none",
|
|
"spanNulls": false,
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"lineInterpolation": "smooth"
|
|
},
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"min": 0
|
|
},
|
|
"overrides": [
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "p50"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "light-blue"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "p95"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "blue"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "p99"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "dark-blue"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"description": "The backend's own response time, as measured by Kong."
|
|
},
|
|
{
|
|
"id": 11,
|
|
"type": "bargauge",
|
|
"title": "Busiest routes",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 23
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "topk(10, sum by (route) (rate(kong_http_requests_total{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval])))",
|
|
"legendFormat": "{{route}}",
|
|
"instant": true
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "reqps",
|
|
"min": 0,
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "continuous-BlPu"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "blue",
|
|
"value": null
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"displayMode": "gradient",
|
|
"orientation": "horizontal",
|
|
"showUnfilled": true,
|
|
"valueMode": "text",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 12,
|
|
"type": "timeseries",
|
|
"title": "Bandwidth by direction",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 23
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum by (direction) (rate(kong_bandwidth_bytes{gateway=~\"(${gateway:pipe})-dev\"}[$__rate_interval]))",
|
|
"legendFormat": "{{direction}}"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "Bps",
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 2,
|
|
"fillOpacity": 8,
|
|
"showPoints": "never",
|
|
"pointSize": 8,
|
|
"gradientMode": "none",
|
|
"spanNulls": false,
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"lineInterpolation": "smooth"
|
|
},
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"min": 0
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 13,
|
|
"type": "row",
|
|
"title": "Audit logs",
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 31
|
|
},
|
|
"panels": []
|
|
},
|
|
{
|
|
"id": 14,
|
|
"type": "timeseries",
|
|
"title": "Audit records by status class",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 32
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "sum by (class) (count_over_time({job=\"kong-audit\", domain=~\"${gateway:pipe}\"} | json status=\"response.status\" | label_format class=`{{ printf \"%.1sxx\" .status }}` [$__auto]))",
|
|
"legendFormat": "{{class}}",
|
|
"queryType": "range"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"min": 0,
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "bars",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 80,
|
|
"showPoints": "never",
|
|
"barAlignment": 0,
|
|
"stacking": {
|
|
"mode": "normal",
|
|
"group": "A"
|
|
},
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto"
|
|
}
|
|
},
|
|
"overrides": [
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "2xx"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "green"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "3xx"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "blue"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "4xx"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "orange"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "5xx"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "red"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"sum"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"description": "The status class is derived from the Kong record with LogQL's json parser, not from a log level: these records carry no severity of their own."
|
|
},
|
|
{
|
|
"id": 15,
|
|
"type": "timeseries",
|
|
"title": "Failing requests by service",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 32
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "sum by (svc) (count_over_time({job=\"kong-audit\", domain=~\"${gateway:pipe}\"} | json status=\"response.status\", svc=\"service.name\" | status =~ \"[45]..\" [$__auto]))",
|
|
"legendFormat": "{{svc}}",
|
|
"queryType": "range"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"min": 0,
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "bars",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 80,
|
|
"showPoints": "never",
|
|
"barAlignment": 0,
|
|
"stacking": {
|
|
"mode": "normal",
|
|
"group": "A"
|
|
},
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto"
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"sum"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"description": "4xx and 5xx only, broken down by the Kong service that answered."
|
|
},
|
|
{
|
|
"id": 16,
|
|
"type": "logs",
|
|
"title": "Recent audit records",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"gridPos": {
|
|
"h": 12,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 40
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "{job=\"kong-audit\", domain=~\"${gateway:pipe}\"} | json",
|
|
"queryType": "range"
|
|
}
|
|
],
|
|
"options": {
|
|
"showTime": true,
|
|
"showLabels": false,
|
|
"showCommonLabels": false,
|
|
"wrapLogMessage": true,
|
|
"prettifyLogMessage": true,
|
|
"sortOrder": "Descending",
|
|
"dedupStrategy": "none",
|
|
"enableLogDetails": true
|
|
},
|
|
"description": "Full Kong record per request: route, status, latencies, client, and the captured request/response bodies. Where the record carries a trace id, the Loki datasource's derived field opens the matching trace in Tempo."
|
|
},
|
|
{
|
|
"id": 17,
|
|
"type": "row",
|
|
"title": "Gateway health",
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 52
|
|
},
|
|
"panels": []
|
|
},
|
|
{
|
|
"id": 18,
|
|
"type": "timeseries",
|
|
"title": "nginx connections",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 53
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum by (state) (kong_nginx_connections_total{gateway=~\"(${gateway:pipe})-dev\"})",
|
|
"legendFormat": "{{state}}"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 2,
|
|
"fillOpacity": 0,
|
|
"showPoints": "never",
|
|
"pointSize": 8,
|
|
"gradientMode": "none",
|
|
"spanNulls": false,
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"lineInterpolation": "smooth"
|
|
},
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"min": 0
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"description": "Active, reading, writing and waiting connections held by the proxy."
|
|
},
|
|
{
|
|
"id": 19,
|
|
"type": "timeseries",
|
|
"title": "Lua shared dict memory",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 53
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum by (shared_dict) (kong_memory_lua_shared_dict_bytes{gateway=~\"(${gateway:pipe})-dev\"})",
|
|
"legendFormat": "{{shared_dict}}"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "bytes",
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 2,
|
|
"fillOpacity": 8,
|
|
"showPoints": "never",
|
|
"pointSize": 8,
|
|
"gradientMode": "none",
|
|
"spanNulls": false,
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"lineInterpolation": "smooth"
|
|
},
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"min": 0
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"description": "Kong keeps rate-limit counters, caches and cluster state in these. A dict filling up silently degrades the plugin that owns it."
|
|
},
|
|
{
|
|
"id": 20,
|
|
"type": "timeseries",
|
|
"title": "Control plane and datastore",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 53
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "min(kong_control_plane_connected{gateway=~\"(${gateway:pipe})-dev\"})",
|
|
"legendFormat": "control plane connected"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "min(kong_datastore_reachable{gateway=~\"(${gateway:pipe})-dev\"})",
|
|
"legendFormat": "datastore reachable"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 2,
|
|
"fillOpacity": 0,
|
|
"showPoints": "never",
|
|
"pointSize": 8,
|
|
"gradientMode": "none",
|
|
"spanNulls": false,
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"lineInterpolation": "smooth"
|
|
},
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"min": 0,
|
|
"decimals": 0
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"description": "1 is healthy. These are dataplanes attached to Konnect, so losing the control plane means config updates stop arriving while traffic keeps flowing on the last known config."
|
|
}
|
|
]
|
|
}
|