Stéphane Tailland
eb35419ad0
One consolidated search compromised on all three questions at once. It also
had a real defect: the Source selector carried `.+` as its All value, so
choosing "All" widened the query back to every stream in the cluster and the
scope preset only ever populated the picker. MongoDB and cert-manager showed up
in what was meant to be an application view.
Each query now carries both `job=~"${source:regex}"` and a literal job matcher,
so the selector actually restricts rather than widens.
The three kinds are subtracted from each other rather than merely filtered:
application logs exclude Keycloak's access-log and events loggers by name, the
Django container's NCSA lines by shape, and the Kong containers' stray access
lines by requiring the nginx date prefix.
Levels are read from the line rather than taken from Loki's detected_level.
That inference marks 78% of the Kong gateway output `unknown` even though every
one of those lines states [notice] or [warn], so filtering on it hid most of
the gateway's own logs. Each parser extracts the level and lowercases it, which
puts nginx's notice, Keycloak's INFO and LibreChat's info on one scale — and
removes the need for an `unknown` entry in the selector.
Applies move to server-side. Client-side apply stores a full copy of each
object in the last-applied-configuration annotation, capped at 256KB, and the
dashboard ConfigMap passed that at eight dashboards; the failure surfaced as
"metadata.annotations: Too long". Server-side apply tracks field ownership in
metadata instead, moving the ceiling to etcd's ~1MB.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
1326 lines
32 KiB
JSON
1326 lines
32 KiB
JSON
{
|
|
"uid": "obs-audit-logs",
|
|
"title": "Audit Logs",
|
|
"description": "Who authenticated, what changed in the realm, and which caller reached which API with which payload. Keycloak identity events and Kong access records, kept apart because they answer different questions.",
|
|
"tags": [
|
|
"observability",
|
|
"logs",
|
|
"audit",
|
|
"security"
|
|
],
|
|
"editable": false,
|
|
"schemaVersion": 39,
|
|
"version": 1,
|
|
"refresh": "",
|
|
"time": {
|
|
"from": "now-24h",
|
|
"to": "now"
|
|
},
|
|
"timezone": "browser",
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"name": "domain",
|
|
"label": "Kong domain",
|
|
"type": "custom",
|
|
"query": "catalog,customers,payments,sales,workforce,llm",
|
|
"multi": true,
|
|
"includeAll": true,
|
|
"allValue": ".+",
|
|
"current": {
|
|
"text": [
|
|
"All"
|
|
],
|
|
"value": [
|
|
"$__all"
|
|
]
|
|
},
|
|
"description": "Applies to the Kong section only."
|
|
},
|
|
{
|
|
"name": "search",
|
|
"label": "Search",
|
|
"type": "textbox",
|
|
"query": "",
|
|
"current": {
|
|
"text": "",
|
|
"value": ""
|
|
},
|
|
"description": "Case-insensitive regex over the whole line. Empty matches everything."
|
|
}
|
|
]
|
|
},
|
|
"panels": [
|
|
{
|
|
"id": 1,
|
|
"type": "row",
|
|
"title": "Identity and administration events (Keycloak)",
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 0
|
|
},
|
|
"panels": []
|
|
},
|
|
{
|
|
"id": 2,
|
|
"type": "stat",
|
|
"title": "Events",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 6,
|
|
"x": 0,
|
|
"y": 1
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "sum(count_over_time({job=\"keycloak/keycloak\"} |~ `(?i)$search` | json logger=\"loggerName\", m=\"message\", lvl=\"level\" | logger=\"org.keycloak.events\" | line_format `{{.m}}` | logfmt [$__range])) or vector(0)",
|
|
"queryType": "instant"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"decimals": 0,
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "fixed",
|
|
"fixedColor": "blue"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "blue",
|
|
"value": null
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"colorMode": "value",
|
|
"graphMode": "none",
|
|
"textMode": "auto"
|
|
}
|
|
},
|
|
{
|
|
"id": 3,
|
|
"type": "stat",
|
|
"title": "Failed logins",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 6,
|
|
"x": 6,
|
|
"y": 1
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "sum(count_over_time({job=\"keycloak/keycloak\"} |~ `(?i)$search` | json logger=\"loggerName\", m=\"message\", lvl=\"level\" | logger=\"org.keycloak.events\" | line_format `{{.m}}` | logfmt | type = \"LOGIN_ERROR\" [$__range])) or vector(0)",
|
|
"queryType": "instant"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"decimals": 0,
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "text",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 1
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"colorMode": "value",
|
|
"graphMode": "none",
|
|
"textMode": "auto"
|
|
},
|
|
"description": "A sustained non-zero value here is the signal a credential-stuffing attempt would leave."
|
|
},
|
|
{
|
|
"id": 4,
|
|
"type": "stat",
|
|
"title": "Successful logins",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 6,
|
|
"x": 12,
|
|
"y": 1
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "sum(count_over_time({job=\"keycloak/keycloak\"} |~ `(?i)$search` | json logger=\"loggerName\", m=\"message\", lvl=\"level\" | logger=\"org.keycloak.events\" | line_format `{{.m}}` | logfmt | type = \"LOGIN\" [$__range])) or vector(0)",
|
|
"queryType": "instant"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"decimals": 0,
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "fixed",
|
|
"fixedColor": "green"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"colorMode": "value",
|
|
"graphMode": "none",
|
|
"textMode": "auto"
|
|
}
|
|
},
|
|
{
|
|
"id": 5,
|
|
"type": "stat",
|
|
"title": "Distinct clients",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 6,
|
|
"x": 18,
|
|
"y": 1
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "count(sum by (clientId) (count_over_time({job=\"keycloak/keycloak\"} |~ `(?i)$search` | json logger=\"loggerName\", m=\"message\", lvl=\"level\" | logger=\"org.keycloak.events\" | line_format `{{.m}}` | logfmt [$__range]))) or vector(0)",
|
|
"queryType": "instant"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"decimals": 0,
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "fixed",
|
|
"fixedColor": "purple"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "purple",
|
|
"value": null
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"colorMode": "value",
|
|
"graphMode": "none",
|
|
"textMode": "auto"
|
|
}
|
|
},
|
|
{
|
|
"id": 6,
|
|
"type": "timeseries",
|
|
"title": "Events by type",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 5
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "sum by (type) (count_over_time({job=\"keycloak/keycloak\"} |~ `(?i)$search` | json logger=\"loggerName\", m=\"message\", lvl=\"level\" | logger=\"org.keycloak.events\" | line_format `{{.m}}` | logfmt [$__auto]))",
|
|
"legendFormat": "{{type}}",
|
|
"queryType": "range"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"min": 0,
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "bars",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 80,
|
|
"showPoints": "never",
|
|
"barAlignment": 0,
|
|
"stacking": {
|
|
"mode": "normal",
|
|
"group": "A"
|
|
}
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"sum"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"description": "LOGIN, LOGIN_ERROR, LOGOUT, REFRESH_TOKEN, CLIENT_LOGIN and the admin events Keycloak raises when a realm object changes."
|
|
},
|
|
{
|
|
"id": 7,
|
|
"type": "timeseries",
|
|
"title": "Events by client",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 5
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "sum by (clientId) (count_over_time({job=\"keycloak/keycloak\"} |~ `(?i)$search` | json logger=\"loggerName\", m=\"message\", lvl=\"level\" | logger=\"org.keycloak.events\" | line_format `{{.m}}` | logfmt [$__auto]))",
|
|
"legendFormat": "{{clientId}}",
|
|
"queryType": "range"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"min": 0,
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "bars",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 80,
|
|
"showPoints": "never",
|
|
"barAlignment": 0,
|
|
"stacking": {
|
|
"mode": "normal",
|
|
"group": "A"
|
|
}
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"sum"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 8,
|
|
"type": "table",
|
|
"title": "Identity events",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"gridPos": {
|
|
"h": 13,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 13
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "{job=\"keycloak/keycloak\"} |~ `(?i)$search` | json logger=\"loggerName\", m=\"message\", lvl=\"level\" | logger=\"org.keycloak.events\" | line_format `{{.m}}` | logfmt",
|
|
"queryType": "range",
|
|
"maxLines": 1000
|
|
}
|
|
],
|
|
"transformations": [
|
|
{
|
|
"id": "extractFields",
|
|
"options": {
|
|
"source": "labels",
|
|
"replace": false
|
|
}
|
|
},
|
|
{
|
|
"id": "merge",
|
|
"options": {}
|
|
},
|
|
{
|
|
"id": "filterFieldsByName",
|
|
"options": {
|
|
"include": {
|
|
"pattern": "^(timestamp|body|type|realmName|clientId|userId|ipAddress|error|grant_type)$"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {},
|
|
"indexByName": {
|
|
"Time": 0,
|
|
"Event": 1,
|
|
"Realm": 2,
|
|
"Client": 3,
|
|
"User": 4,
|
|
"Source IP": 5,
|
|
"Error": 6,
|
|
"Grant": 7,
|
|
"Raw line": 8
|
|
},
|
|
"renameByName": {
|
|
"timestamp": "Time",
|
|
"body": "Raw line",
|
|
"type": "Event",
|
|
"realmName": "Realm",
|
|
"clientId": "Client",
|
|
"userId": "User",
|
|
"ipAddress": "Source IP",
|
|
"error": "Error",
|
|
"grant_type": "Grant"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"align": "auto",
|
|
"cellOptions": {
|
|
"type": "auto"
|
|
},
|
|
"inspect": true,
|
|
"filterable": true
|
|
},
|
|
"mappings": []
|
|
},
|
|
"overrides": [
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "Event"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "custom.width",
|
|
"value": 180
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "Realm"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "custom.width",
|
|
"value": 150
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "Client"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "custom.width",
|
|
"value": 170
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "User"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "custom.width",
|
|
"value": 260
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "Source IP"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "custom.width",
|
|
"value": 130
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "Error"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "custom.width",
|
|
"value": 220
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "Grant"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "custom.width",
|
|
"value": 120
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "Raw line"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "custom.width",
|
|
"value": 240
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"options": {
|
|
"showHeader": true,
|
|
"cellHeight": "sm",
|
|
"footer": {
|
|
"show": false,
|
|
"reducer": [
|
|
"sum"
|
|
],
|
|
"countRows": false,
|
|
"fields": ""
|
|
},
|
|
"sortBy": [
|
|
{
|
|
"displayName": "Time",
|
|
"desc": true
|
|
}
|
|
]
|
|
},
|
|
"description": "Keycloak writes these as key=\"value\" pairs inside its JSON message field, so the line is reformatted to the message and then read as logfmt. A userId of \"null\" means the subject was never resolved — the usual shape of a failed authentication."
|
|
},
|
|
{
|
|
"id": 9,
|
|
"type": "row",
|
|
"title": "API access records (Kong)",
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 26
|
|
},
|
|
"panels": []
|
|
},
|
|
{
|
|
"id": 10,
|
|
"type": "stat",
|
|
"title": "Records",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 6,
|
|
"x": 0,
|
|
"y": 27
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "sum(count_over_time({job=\"kong-audit\", domain=~\"${domain:regex}\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", svc=\"service.name\", route=\"route.name\", authz=\"request.headers.authorization\" [$__range])) or vector(0)",
|
|
"queryType": "instant"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"decimals": 0,
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "fixed",
|
|
"fixedColor": "blue"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "blue",
|
|
"value": null
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"colorMode": "value",
|
|
"graphMode": "none",
|
|
"textMode": "auto"
|
|
}
|
|
},
|
|
{
|
|
"id": 11,
|
|
"type": "stat",
|
|
"title": "Rejected (401/403)",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 6,
|
|
"x": 6,
|
|
"y": 27
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "sum(count_over_time({job=\"kong-audit\", domain=~\"${domain:regex}\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", svc=\"service.name\", route=\"route.name\", authz=\"request.headers.authorization\" | status =~ \"40[13]\" [$__range])) or vector(0)",
|
|
"queryType": "instant"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"decimals": 0,
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "text",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "orange",
|
|
"value": 1
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"colorMode": "value",
|
|
"graphMode": "none",
|
|
"textMode": "auto"
|
|
}
|
|
},
|
|
{
|
|
"id": 12,
|
|
"type": "stat",
|
|
"title": "Distinct callers",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 6,
|
|
"x": 12,
|
|
"y": 27
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "count(sum by (client) (count_over_time({job=\"kong-audit\", domain=~\"${domain:regex}\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", svc=\"service.name\", route=\"route.name\", authz=\"request.headers.authorization\" [$__range]))) or vector(0)",
|
|
"queryType": "instant"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"decimals": 0,
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "fixed",
|
|
"fixedColor": "purple"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "purple",
|
|
"value": null
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"colorMode": "value",
|
|
"graphMode": "none",
|
|
"textMode": "auto"
|
|
}
|
|
},
|
|
{
|
|
"id": 13,
|
|
"type": "stat",
|
|
"title": "Domains covered",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 6,
|
|
"x": 18,
|
|
"y": 27
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "count(sum by (domain) (count_over_time({job=\"kong-audit\", domain=~\"${domain:regex}\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", svc=\"service.name\", route=\"route.name\", authz=\"request.headers.authorization\" [$__range]))) or vector(0)",
|
|
"queryType": "instant"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"decimals": 0,
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "fixed",
|
|
"fixedColor": "text"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "text",
|
|
"value": null
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"colorMode": "value",
|
|
"graphMode": "none",
|
|
"textMode": "auto"
|
|
}
|
|
},
|
|
{
|
|
"id": 14,
|
|
"type": "timeseries",
|
|
"title": "Records by domain",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 31
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "sum by (domain) (count_over_time({job=\"kong-audit\", domain=~\"${domain:regex}\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", svc=\"service.name\", route=\"route.name\", authz=\"request.headers.authorization\" [$__auto]))",
|
|
"legendFormat": "{{domain}}",
|
|
"queryType": "range"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"min": 0,
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "bars",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 80,
|
|
"showPoints": "never",
|
|
"barAlignment": 0,
|
|
"stacking": {
|
|
"mode": "normal",
|
|
"group": "A"
|
|
}
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"sum"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 15,
|
|
"type": "timeseries",
|
|
"title": "Outcome by status class",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 31
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "sum by (class) (count_over_time({job=\"kong-audit\", domain=~\"${domain:regex}\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", svc=\"service.name\", route=\"route.name\", authz=\"request.headers.authorization\" | label_format class=`{{ printf \"%.1sxx\" .status }}` [$__auto]))",
|
|
"legendFormat": "{{class}}",
|
|
"queryType": "range"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"min": 0,
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "bars",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 80,
|
|
"showPoints": "never",
|
|
"barAlignment": 0,
|
|
"stacking": {
|
|
"mode": "normal",
|
|
"group": "A"
|
|
}
|
|
}
|
|
},
|
|
"overrides": [
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "2xx"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "green"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "3xx"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "blue"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "4xx"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "orange"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "5xx"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "red"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"sum"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 16,
|
|
"type": "table",
|
|
"title": "API access records",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"gridPos": {
|
|
"h": 13,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 39
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "{job=\"kong-audit\", domain=~\"${domain:regex}\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", svc=\"service.name\", route=\"route.name\", authz=\"request.headers.authorization\"",
|
|
"queryType": "range",
|
|
"maxLines": 1000
|
|
}
|
|
],
|
|
"transformations": [
|
|
{
|
|
"id": "extractFields",
|
|
"options": {
|
|
"source": "labels",
|
|
"replace": false
|
|
}
|
|
},
|
|
{
|
|
"id": "merge",
|
|
"options": {}
|
|
},
|
|
{
|
|
"id": "filterFieldsByName",
|
|
"options": {
|
|
"include": {
|
|
"pattern": "^(timestamp|body|domain|svc|route|method|path|status|client)$"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {},
|
|
"indexByName": {
|
|
"Time": 0,
|
|
"Domain": 1,
|
|
"Service": 2,
|
|
"Route": 3,
|
|
"Method": 4,
|
|
"Path": 5,
|
|
"Status": 6,
|
|
"Caller IP": 7,
|
|
"Raw line": 8
|
|
},
|
|
"renameByName": {
|
|
"timestamp": "Time",
|
|
"body": "Raw line",
|
|
"domain": "Domain",
|
|
"svc": "Service",
|
|
"route": "Route",
|
|
"method": "Method",
|
|
"path": "Path",
|
|
"status": "Status",
|
|
"client": "Caller IP"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"align": "auto",
|
|
"cellOptions": {
|
|
"type": "auto"
|
|
},
|
|
"inspect": true,
|
|
"filterable": true
|
|
},
|
|
"mappings": []
|
|
},
|
|
"overrides": [
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "Domain"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "custom.width",
|
|
"value": 120
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "Service"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "custom.width",
|
|
"value": 150
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "Route"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "custom.width",
|
|
"value": 230
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "Method"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "custom.width",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "Status"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "custom.width",
|
|
"value": 80
|
|
},
|
|
{
|
|
"id": "mappings",
|
|
"value": [
|
|
{
|
|
"type": "regex",
|
|
"options": {
|
|
"pattern": "^2..$",
|
|
"result": {
|
|
"color": "green",
|
|
"index": 0
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "regex",
|
|
"options": {
|
|
"pattern": "^3..$",
|
|
"result": {
|
|
"color": "blue",
|
|
"index": 1
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "regex",
|
|
"options": {
|
|
"pattern": "^4..$",
|
|
"result": {
|
|
"color": "orange",
|
|
"index": 2
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "regex",
|
|
"options": {
|
|
"pattern": "^5..$",
|
|
"result": {
|
|
"color": "red",
|
|
"index": 3
|
|
}
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": "custom.cellOptions",
|
|
"value": {
|
|
"type": "color-text"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "Caller IP"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "custom.width",
|
|
"value": 130
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "Raw line"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "custom.width",
|
|
"value": 240
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"options": {
|
|
"showHeader": true,
|
|
"cellHeight": "sm",
|
|
"footer": {
|
|
"show": false,
|
|
"reducer": [
|
|
"sum"
|
|
],
|
|
"countRows": false,
|
|
"fields": ""
|
|
},
|
|
"sortBy": [
|
|
{
|
|
"displayName": "Time",
|
|
"desc": true
|
|
}
|
|
]
|
|
},
|
|
"description": "The Raw line column holds the complete record, request and response bodies included — that is the evidentiary part, and the reason these are kept beyond what an access log needs. Note the captured Authorization header is a bearer token in clear text: acceptable for this lab, not for anything else."
|
|
}
|
|
]
|
|
}
|