{ "uid": "obs-classic-models-api", "title": "Classic Models API — Overview", "description": "HTTP traffic, per-view latency, MySQL access and logs for the Django API. django-prometheus metrics aggregated across gunicorn workers.", "tags": [ "observability", "classic-models-api", "django", "application", "metrics", "logs" ], "editable": false, "schemaVersion": 39, "version": 1, "refresh": "30s", "time": { "from": "now-6h", "to": "now" }, "timezone": "browser", "templating": { "list": [] }, "panels": [ { "id": 1, "type": "row", "title": "Overview", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "panels": [] }, { "id": 2, "type": "stat", "title": "Requests / s", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 5, "w": 6, "x": 0, "y": 1 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(django_http_requests_total_by_method_total{namespace=\"classic-models\"}[$__rate_interval]))", "instant": false } ], "fieldConfig": { "defaults": { "unit": "reqps", "mappings": [], "decimals": 2, "color": { "mode": "fixed", "fixedColor": "blue" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "blue", "value": null } ] } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "value", "graphMode": "area", "textMode": "auto", "justifyMode": "auto" }, "description": "Aggregated across both replicas and the 3 gunicorn workers of each (prometheus_client multiprocess mode)." }, { "id": 3, "type": "stat", "title": "Error rate", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 5, "w": 6, "x": 6, "y": 1 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "(sum(rate(django_http_responses_total_by_status_total{namespace=\"classic-models\",status=~\"5..\"}[$__rate_interval])) or vector(0)) / clamp_min(sum(rate(django_http_responses_total_by_status_total{namespace=\"classic-models\"}[$__rate_interval])), 0.001)", "instant": false } ], "fieldConfig": { "defaults": { "unit": "percentunit", "mappings": [], "decimals": 2, "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 0.01 }, { "color": "red", "value": 0.05 } ] } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "value", "graphMode": "area", "textMode": "auto", "justifyMode": "auto" }, "description": "5xx only: the API returns expected 401/404 that must not count as outages. `or vector(0)` is required: with no errors at all the numerator is an empty vector, and empty/x stays empty -- the panel would read \"No data\" instead of 0%. clamp_min covers the zero-traffic case." }, { "id": 4, "type": "stat", "title": "Latency p95", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 5, "w": 6, "x": 12, "y": 1 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "histogram_quantile(0.95, sum by (le) (rate(django_http_requests_latency_seconds_by_view_method_bucket{namespace=\"classic-models\"}[$__rate_interval])))", "instant": false } ], "fieldConfig": { "defaults": { "unit": "s", "mappings": [], "decimals": 3, "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 0.5 }, { "color": "red", "value": 2 } ] } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "value", "graphMode": "area", "textMode": "auto", "justifyMode": "auto" }, "description": "Measured between django-prometheus' Before and After middleware, which bracket the whole stack." }, { "id": 5, "type": "stat", "title": "SQL queries / s", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 5, "w": 6, "x": 18, "y": 1 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(django_db_execute_total{namespace=\"classic-models\"}[$__rate_interval]))", "instant": false } ], "fieldConfig": { "defaults": { "unit": "reqps", "mappings": [], "decimals": 2, "color": { "mode": "fixed", "fixedColor": "purple" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "purple", "value": null } ] } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "value", "graphMode": "area", "textMode": "auto", "justifyMode": "auto" } }, { "id": 6, "type": "row", "title": "HTTP traffic", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 6 }, "panels": [] }, { "id": 7, "type": "timeseries", "title": "Responses by status class", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 7 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (class) (label_replace(rate(django_http_responses_total_by_status_total{namespace=\"classic-models\"}[$__rate_interval]), \"class\", \"$1xx\", \"status\", \"(.)..\"))", "legendFormat": "{{class}}" } ], "fieldConfig": { "defaults": { "unit": "reqps", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 70, "showPoints": "never", "pointSize": 8, "gradientMode": "none", "spanNulls": false, "axisLabel": "", "axisPlacement": "auto", "lineInterpolation": "linear", "stacking": { "mode": "normal", "group": "A" } }, "mappings": [], "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "2xx" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } } ] }, { "matcher": { "id": "byName", "options": "3xx" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } } ] }, { "matcher": { "id": "byName", "options": "4xx" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "orange" } } ] }, { "matcher": { "id": "byName", "options": "5xx" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } ] }, { "matcher": { "id": "byName", "options": "SUCCESS" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } } ] }, { "matcher": { "id": "byName", "options": "REDIRECTION" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } } ] }, { "matcher": { "id": "byName", "options": "CLIENT_ERROR" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "orange" } } ] }, { "matcher": { "id": "byName", "options": "SERVER_ERROR" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } ] } ] }, "options": { "legend": { "showLegend": true, "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 8, "type": "timeseries", "title": "Request latency", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 7 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "histogram_quantile(0.50, sum by (le) (rate(django_http_requests_latency_seconds_by_view_method_bucket{namespace=\"classic-models\"}[$__rate_interval])))", "legendFormat": "p50" }, { "refId": "B", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "histogram_quantile(0.95, sum by (le) (rate(django_http_requests_latency_seconds_by_view_method_bucket{namespace=\"classic-models\"}[$__rate_interval])))", "legendFormat": "p95" }, { "refId": "C", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "histogram_quantile(0.99, sum by (le) (rate(django_http_requests_latency_seconds_by_view_method_bucket{namespace=\"classic-models\"}[$__rate_interval])))", "legendFormat": "p99" } ], "fieldConfig": { "defaults": { "unit": "s", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "pointSize": 8, "gradientMode": "none", "spanNulls": false, "axisLabel": "", "axisPlacement": "auto", "lineInterpolation": "smooth" }, "mappings": [], "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "p50" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "light-blue" } } ] }, { "matcher": { "id": "byName", "options": "p95" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } } ] }, { "matcher": { "id": "byName", "options": "p99" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "dark-blue" } } ] } ] }, "options": { "legend": { "showLegend": true, "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 9, "type": "bargauge", "title": "Busiest views", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 15 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "topk(10, sum by (view) (rate(django_http_requests_latency_seconds_by_view_method_count{namespace=\"classic-models\"}[$__rate_interval])))", "legendFormat": "{{view}}", "instant": true } ], "fieldConfig": { "defaults": { "unit": "reqps", "min": 0, "mappings": [], "color": { "mode": "continuous-BlPu" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "blue", "value": null } ] } }, "overrides": [] }, "options": { "displayMode": "gradient", "orientation": "horizontal", "showUnfilled": true, "valueMode": "text", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } } }, { "id": 10, "type": "timeseries", "title": "p95 latency per view", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 15 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "topk(10, histogram_quantile(0.95, sum by (view, le) (rate(django_http_requests_latency_seconds_by_view_method_bucket{namespace=\"classic-models\"}[$__rate_interval]))))", "legendFormat": "{{view}}" } ], "fieldConfig": { "defaults": { "unit": "s", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "pointSize": 8, "gradientMode": "none", "spanNulls": false, "axisLabel": "", "axisPlacement": "auto", "lineInterpolation": "smooth" }, "mappings": [], "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "Isolates the view responsible for a regression in the global p95." }, { "id": 11, "type": "timeseries", "title": "Requests by HTTP method", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 23 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (method) (rate(django_http_requests_total_by_method_total{namespace=\"classic-models\"}[$__rate_interval]))", "legendFormat": "{{method}}" } ], "fieldConfig": { "defaults": { "unit": "reqps", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 8, "showPoints": "never", "pointSize": 8, "gradientMode": "none", "spanNulls": false, "axisLabel": "", "axisPlacement": "auto", "lineInterpolation": "smooth" }, "mappings": [], "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 12, "type": "timeseries", "title": "Response volume", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 23 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(django_http_responses_body_total_bytes_sum{namespace=\"classic-models\"}[$__rate_interval]))", "legendFormat": "bytes/s" } ], "fieldConfig": { "defaults": { "unit": "Bps", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 8, "showPoints": "never", "pointSize": 8, "gradientMode": "none", "spanNulls": false, "axisLabel": "", "axisPlacement": "auto", "lineInterpolation": "smooth" }, "mappings": [], "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": false, "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "Application egress, useful to spot missing pagination or an over-eager serializer." }, { "id": 13, "type": "row", "title": "MySQL database", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 30 }, "panels": [] }, { "id": 14, "type": "timeseries", "title": "SQL queries executed", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 31 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (vendor) (rate(django_db_execute_total{namespace=\"classic-models\"}[$__rate_interval]))", "legendFormat": "{{vendor}}" } ], "fieldConfig": { "defaults": { "unit": "reqps", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 8, "showPoints": "never", "pointSize": 8, "gradientMode": "none", "spanNulls": false, "axisLabel": "", "axisPlacement": "auto", "lineInterpolation": "smooth" }, "mappings": [], "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 15, "type": "timeseries", "title": "SQL query latency", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 31 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "histogram_quantile(0.50, sum by (le) (rate(django_db_query_duration_seconds_bucket{namespace=\"classic-models\"}[$__rate_interval])))", "legendFormat": "p50" }, { "refId": "B", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "histogram_quantile(0.95, sum by (le) (rate(django_db_query_duration_seconds_bucket{namespace=\"classic-models\"}[$__rate_interval])))", "legendFormat": "p95" }, { "refId": "C", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "histogram_quantile(0.99, sum by (le) (rate(django_db_query_duration_seconds_bucket{namespace=\"classic-models\"}[$__rate_interval])))", "legendFormat": "p99" } ], "fieldConfig": { "defaults": { "unit": "s", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "pointSize": 8, "gradientMode": "none", "spanNulls": false, "axisLabel": "", "axisPlacement": "auto", "lineInterpolation": "smooth" }, "mappings": [], "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "p50" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "light-blue" } } ] }, { "matcher": { "id": "byName", "options": "p95" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } } ] }, { "matcher": { "id": "byName", "options": "p99" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "dark-blue" } } ] } ] }, "options": { "legend": { "showLegend": true, "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 16, "type": "timeseries", "title": "New DB connections", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 31 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(django_db_new_connections_total{namespace=\"classic-models\"}[$__rate_interval]))", "legendFormat": "connections/s" } ], "fieldConfig": { "defaults": { "unit": "reqps", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 8, "showPoints": "never", "pointSize": 8, "gradientMode": "none", "spanNulls": false, "axisLabel": "", "axisPlacement": "auto", "lineInterpolation": "smooth" }, "mappings": [], "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": false, "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "Django opens one connection per request while CONN_MAX_AGE stays at 0. A rate close to the HTTP throughput confirms there is no connection reuse." }, { "id": 17, "type": "row", "title": "Logs", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 38 }, "panels": [] }, { "id": 18, "type": "timeseries", "title": "Log volume by level", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 39 }, "targets": [ { "refId": "A", "datasource": { "type": "loki", "uid": "loki" }, "expr": "sum by (detected_level) (count_over_time({namespace=\"classic-models\",container=\"classic-models-api\"}[$__auto]))", "legendFormat": "{{detected_level}}", "queryType": "range" } ], "fieldConfig": { "defaults": { "unit": "short", "min": 0, "mappings": [], "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "bars", "lineWidth": 1, "fillOpacity": 80, "showPoints": "never", "barAlignment": 0, "stacking": { "mode": "normal", "group": "A" }, "axisLabel": "", "axisPlacement": "auto" } }, "overrides": [ { "matcher": { "id": "byName", "options": "critical" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "dark-red" } } ] }, { "matcher": { "id": "byName", "options": "error" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } ] }, { "matcher": { "id": "byName", "options": "warn" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "orange" } } ] }, { "matcher": { "id": "byName", "options": "info" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } } ] }, { "matcher": { "id": "byName", "options": "debug" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } } ] }, { "matcher": { "id": "byName", "options": "unknown" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "text" } } ] } ] }, "options": { "legend": { "showLegend": true, "displayMode": "table", "placement": "bottom", "calcs": [ "sum" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "`detected_level` is inferred by Loki at ingestion, so one query works across every log format." }, { "id": 19, "type": "timeseries", "title": "Errors by container (whole namespace)", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 39 }, "targets": [ { "refId": "A", "datasource": { "type": "loki", "uid": "loki" }, "expr": "sum by (container) (count_over_time({namespace=\"classic-models\"} | detected_level =~ \"error|critical\" [$__auto]))", "legendFormat": "{{container}}", "queryType": "range" } ], "fieldConfig": { "defaults": { "unit": "short", "min": 0, "mappings": [], "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "bars", "lineWidth": 1, "fillOpacity": 80, "showPoints": "never", "barAlignment": 0, "stacking": { "mode": "normal", "group": "A" }, "axisLabel": "", "axisPlacement": "auto" } }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "table", "placement": "bottom", "calcs": [ "sum" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "Namespace-wide so the datastore's failures show up next to the application's." }, { "id": 25, "type": "table", "title": "Access log", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 11, "w": 24, "x": 0, "y": 47 }, "targets": [ { "refId": "A", "datasource": { "type": "loki", "uid": "loki" }, "expr": "{namespace=\"classic-models\", container=\"classic-models-api\"} | pattern ` - - [] \" \" <_> \"\"`", "queryType": "range", "maxLines": 500 } ], "transformations": [ { "id": "extractFields", "options": { "source": "labels", "replace": false } }, { "id": "filterFieldsByName", "options": { "include": { "pattern": "^(timestamp|body|status|method|uri|size|client|agent)$" } } }, { "id": "organize", "options": { "excludeByName": {}, "indexByName": { "Time": 0, "status": 1, "method": 2, "uri": 3, "size": 4, "client": 5, "agent": 6, "body": 7 }, "renameByName": { "status": "Status", "method": "Method", "uri": "URI", "size": "Bytes", "client": "Client", "agent": "User agent", "body": "Raw record", "timestamp": "Time" } } } ], "fieldConfig": { "defaults": { "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": true, "filterable": true }, "mappings": [] }, "overrides": [ { "matcher": { "id": "byName", "options": "Status" }, "properties": [ { "id": "custom.width", "value": 80 }, { "id": "mappings", "value": [ { "type": "regex", "options": { "pattern": "^2..$", "result": { "color": "green", "index": 0 } } }, { "type": "regex", "options": { "pattern": "^3..$", "result": { "color": "blue", "index": 1 } } }, { "type": "regex", "options": { "pattern": "^4..$", "result": { "color": "orange", "index": 2 } } }, { "type": "regex", "options": { "pattern": "^5..$", "result": { "color": "red", "index": 3 } } } ] }, { "id": "custom.cellOptions", "value": { "type": "color-text" } } ] }, { "matcher": { "id": "byName", "options": "Method" }, "properties": [ { "id": "custom.width", "value": 80 } ] }, { "matcher": { "id": "byName", "options": "URI" }, "properties": [ { "id": "custom.width", "value": 300 } ] }, { "matcher": { "id": "byName", "options": "Bytes" }, "properties": [ { "id": "custom.width", "value": 90 } ] }, { "matcher": { "id": "byName", "options": "Client" }, "properties": [ { "id": "custom.width", "value": 130 } ] }, { "matcher": { "id": "byName", "options": "User agent" }, "properties": [ { "id": "custom.width", "value": 200 } ] }, { "matcher": { "id": "byName", "options": "Raw record" }, "properties": [ { "id": "custom.width", "value": 260 } ] } ] }, "options": { "showHeader": true, "cellHeight": "sm", "footer": { "show": false, "reducer": [ "sum" ], "countRows": false, "fields": "" }, "sortBy": [ { "displayName": "Time", "desc": true } ] }, "description": "gunicorn writes NCSA access lines, not JSON, so these are split by pattern. Lines that do not match — application JSON logs — are kept with empty columns rather than dropped." }, { "id": 26, "type": "row", "title": "Raw log lines", "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 58 }, "panels": [ { "id": 20, "type": "logs", "title": "Raw log lines", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 11, "w": 24, "x": 0, "y": 59 }, "targets": [ { "refId": "A", "datasource": { "type": "loki", "uid": "loki" }, "expr": "{namespace=\"classic-models\"} | detected_level =~ \"error|critical|warn\"", "queryType": "range" } ], "options": { "showTime": true, "showLabels": false, "showCommonLabels": false, "wrapLogMessage": true, "prettifyLogMessage": false, "sortOrder": "Descending", "dedupStrategy": "none", "enableLogDetails": true }, "description": "Kept alongside the table: the logs panel renders ANSI colour, expands a record's detail inline, tails live, and turns a trace id into a link to Tempo — none of which a table can do. Collapsed so it stays out of the way until needed." } ] }, { "id": 21, "type": "row", "title": "Container resources", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 59 }, "panels": [] }, { "id": 22, "type": "timeseries", "title": "CPU per container", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 60 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (pod, container) (rate(container_cpu_usage_seconds_total{namespace=\"classic-models\",pod=~\"classic-models-api-.*\",container!=\"\"}[$__rate_interval]))", "legendFormat": "{{pod}}/{{container}}" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 8, "showPoints": "never", "pointSize": 8, "gradientMode": "none", "spanNulls": false, "axisLabel": "", "axisPlacement": "auto", "lineInterpolation": "smooth" }, "mappings": [], "color": { "mode": "palette-classic" }, "min": 0, "decimals": 3 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "Cores consumed." }, { "id": 23, "type": "timeseries", "title": "Memory per container", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 60 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (pod, container) (container_memory_working_set_bytes{namespace=\"classic-models\",pod=~\"classic-models-api-.*\",container!=\"\"})", "legendFormat": "{{pod}}/{{container}}" } ], "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 8, "showPoints": "never", "pointSize": 8, "gradientMode": "none", "spanNulls": false, "axisLabel": "", "axisPlacement": "auto", "lineInterpolation": "smooth" }, "mappings": [], "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "Working set: what the kernel cannot reclaim under pressure. This is the figure the OOM killer compares against the limit." }, { "id": 24, "type": "timeseries", "title": "Restarts over the window", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 60 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (pod) (changes(container_start_time_seconds{namespace=\"classic-models\",pod=~\"classic-models-api-.*\",container!=\"\"}[$__range]))", "legendFormat": "{{pod}}" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "pointSize": 8, "gradientMode": "none", "spanNulls": false, "axisLabel": "", "axisPlacement": "auto", "lineInterpolation": "smooth" }, "mappings": [], "color": { "mode": "palette-classic" }, "min": 0, "decimals": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "Container start count over the displayed window. Stays at 0 in nominal operation." } ] }