{
  "uid": "obs-distributed-traces",
  "title": "Distributed Traces",
  "description": "Find a trace, and keep only the spans worth looking at. Built because Explore's unfiltered search is unusable here: Tempo does not return {} newest-first, so health checks crowd out real transactions.\n\nDeliberately carries no metric panels. A span's own duration is in the span tables, measured; aggregate RED by span name is what the per-application dashboards already show, and neither answers 'the metrics of this span'.",
  "tags": [
    "observability",
    "tracing",
    "tempo"
  ],
  "timezone": "browser",
  "schemaVersion": 39,
  "version": 1,
  "refresh": "",
  "time": {
    "from": "now-1h",
    "to": "now"
  },
  "editable": true,
  "graphTooltip": 1,
  "templating": {
    "list": [
      {
        "name": "service",
        "label": "Service",
        "type": "query",
        "datasource": {
          "type": "prometheus",
          "uid": "prometheus"
        },
        "query": {
          "query": "label_values(traces_spanmetrics_calls_total, service)",
          "refId": "service"
        },
        "refresh": 2,
        "multi": true,
        "includeAll": true,
        "allValue": ".+",
        "current": {
          "text": [
            "All"
          ],
          "value": [
            "$__all"
          ]
        },
        "description": "Services known to Tempo's span metrics. Drives the trace, span and metric panels."
      },
      {
        "name": "traceId",
        "label": "Trace ID",
        "type": "textbox",
        "query": "",
        "current": {
          "text": "",
          "value": ""
        },
        "description": "Set by clicking a Trace ID in the top table, or paste one. Empty means the log filter matches everything."
      }
    ]
  },
  "panels": [
    {
      "type": "row",
      "title": "Traces",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "collapsed": false,
      "panels": []
    },
    {
      "type": "table",
      "title": "Traces on the dashboard time range",
      "description": "TraceQL over the dashboard's time range. Filtered by service so the health-check noise stays out: an unfiltered {} query is NOT returned newest-first by Tempo and the limit truncates arbitrarily, which is why browsing raw Explore misses recent traces. Click a Trace ID to load it below.",
      "datasource": {
        "type": "tempo",
        "uid": "tempo"
      },
      "gridPos": {
        "h": 9,
        "w": 24,
        "x": 0,
        "y": 1
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "tempo",
            "uid": "tempo"
          },
          "queryType": "traceql",
          "query": "{resource.service.name=~\"$service\"}",
          "tableType": "traces",
          "limit": 50,
          "spss": 20
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": "auto",
            "filterable": true
          }
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "traceIdHidden"
            },
            "properties": [
              {
                "id": "custom.hidden",
                "value": true
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "traceID"
            },
            "properties": [
              {
                "id": "links",
                "value": [
                  {
                    "title": "Load this trace below",
                    "url": "d/obs-distributed-traces/distributed-traces?${__url_time_range}&var-traceId=${__value.raw}&var-service=${service:queryparam}"
                  }
                ]
              }
            ]
          }
        ]
      },
      "options": {
        "showHeader": true,
        "sortBy": [
          {
            "displayName": "Start time",
            "desc": true
          }
        ]
      }
    },
    {
      "type": "row",
      "title": "Spans that matter",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 10
      },
      "collapsed": false,
      "panels": []
    },
    {
      "type": "table",
      "title": "HTTP hops between components",
      "description": "One row per component entry: these are the SERVER-kind spans, i.e. the inbound HTTP request each service received. Browser to LibreChat, LibreChat to the llm gateway, gateway to a domain gateway, gateway to the backend. Everything else — express middleware, DNS, SQL — is deliberately excluded.",
      "datasource": {
        "type": "tempo",
        "uid": "tempo"
      },
      "gridPos": {
        "h": 10,
        "w": 12,
        "x": 0,
        "y": 11
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "tempo",
            "uid": "tempo"
          },
          "queryType": "traceql",
          "query": "{resource.service.name=~\"$service\" && kind=server} | select(span.http.method, span.http.status_code, span.http.route, span.http.url)",
          "tableType": "spans",
          "limit": 50,
          "spss": 20
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": "auto",
            "filterable": true
          }
        },
        "overrides": []
      },
      "options": {
        "showHeader": true
      }
    },
    {
      "type": "table",
      "title": "Kong plugin execution",
      "description": "Every plugin Kong ran, with its own duration — openid-connect, rate-limiting, request-transformer, opentelemetry, and so on. This is what shows which plugin is costing latency on a gateway hop.",
      "datasource": {
        "type": "tempo",
        "uid": "tempo"
      },
      "gridPos": {
        "h": 10,
        "w": 12,
        "x": 12,
        "y": 11
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "tempo",
            "uid": "tempo"
          },
          "queryType": "traceql",
          "query": "{resource.service.name=~\"$service\" && name=~\"kong\\\\..*plugin.*\"}",
          "tableType": "spans",
          "limit": 50,
          "spss": 20
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": "auto",
            "filterable": true
          }
        },
        "overrides": []
      },
      "options": {
        "showHeader": true
      }
    },
    {
      "type": "row",
      "title": "Logs for the selected trace",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 21
      },
      "collapsed": false,
      "panels": []
    },
    {
      "type": "logs",
      "title": "Logs carrying trace $traceId",
      "description": "Correlation is by TRACE id, not span id: none of the emitters log a span id, so this is every log line of the whole trace rather than of one span.\n\nCoverage is uneven and worth knowing: kong-audit and the backend's access log carry the id on every line; the backend's application log does too since v1.0.4. LibreChat logs carry no trace id at all, so its lines never appear here.\n\nWith traceId empty the filter matches everything — paste an id or click one above.",
      "datasource": {
        "type": "loki",
        "uid": "loki"
      },
      "gridPos": {
        "h": 11,
        "w": 24,
        "x": 0,
        "y": 22
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "loki",
            "uid": "loki"
          },
          "expr": "{job=~\"kong-audit|classic-models/classic-models-api|librechat/librechat|keycloak/keycloak\"} |= \"$traceId\""
        }
      ],
      "options": {
        "showTime": true,
        "wrapLogMessage": true,
        "sortOrder": "Ascending",
        "enableLogDetails": true
      }
    }
  ]
}