# =============================================================================
#  infra-observability -- Grafana / Prometheus / Loki / Tempo stack (+ Alloy)
#  Fully declarative: kustomize + kubectl. No imperative `helm install`.
#
#  make            -> help
#  make deploy     -> deploy everything
#  make diag       -> full diagnostics
# =============================================================================

SHELL       := /bin/bash
.DEFAULT_GOAL := help
# Every target prints its own output, so commands are not echoed.
.SILENT:

NS          ?= observability
CTX         ?= orbstack
KUBECTL     := kubectl --context=$(CTX)
K8S         := k8s
DASHBOARDS  := $(K8S)/grafana/dashboards
INGRESS_IP  ?= 192.168.139.101
HOSTS       := grafana.sttlab.local prometheus.sttlab.local alloy.sttlab.local

# Components backed by a PVC (this is also the flush order)
STATEFUL    := prometheus loki tempo grafana
# Restartable workloads
WORKLOADS   := deployment/prometheus deployment/loki deployment/tempo deployment/grafana daemonset/alloy

C_OK   := \033[0;32m
C_WARN := \033[0;33m
C_ERR  := \033[0;31m
C_HEAD := \033[1;36m
C_OFF  := \033[0m

# -----------------------------------------------------------------------------
# Help
# -----------------------------------------------------------------------------
.PHONY: help
help: ## Show this help
	printf "$(C_HEAD)infra-observability$(C_OFF) -- namespace $(C_OK)$(NS)$(C_OFF) / context $(C_OK)$(CTX)$(C_OFF)\n\n"
	awk 'BEGIN {FS = ":.*?## "} \
		/^[a-zA-Z0-9_-]+:.*?## / {printf "  $(C_OK)%-24s$(C_OFF) %s\n", $$1, $$2} \
		/^##@/ {printf "\n$(C_HEAD)%s$(C_OFF)\n", substr($$0, 5)}' $(MAKEFILE_LIST)
	printf "\n"

##@ Deployment

.PHONY: deploy
deploy: check-context secret ## Deploy the whole stack (runtimes + dashboards)
	$(KUBECTL) apply -k $(K8S)
	$(MAKE) --no-print-directory wait

.PHONY: deploy-runtime
deploy-runtime: check-context secret ## Deploy runtimes only (no dashboards)
	$(KUBECTL) apply -f $(K8S)/namespace.yaml -f $(K8S)/secret.yaml
	$(KUBECTL) apply -k $(K8S)/prometheus
	$(KUBECTL) apply -k $(K8S)/loki
	$(KUBECTL) apply -k $(K8S)/tempo
	$(KUBECTL) apply -k $(K8S)/alloy
	$(KUBECTL) apply -k $(K8S)/grafana
	$(KUBECTL) apply -k $(K8S)/ingress
	$(MAKE) --no-print-directory wait

.PHONY: prometheus loki tempo grafana alloy ingress
prometheus: check-context ## Deploy/update Prometheus only
	$(KUBECTL) apply -k $(K8S)/prometheus
loki: check-context ## Deploy/update Loki only
	$(KUBECTL) apply -k $(K8S)/loki
tempo: check-context ## Deploy/update Tempo only
	$(KUBECTL) apply -k $(K8S)/tempo
grafana: check-context ## Deploy/update the Grafana runtime only (datasources included)
	$(KUBECTL) apply -k $(K8S)/grafana
alloy: check-context ## Deploy/update Alloy only (log collection)
	$(KUBECTL) apply -k $(K8S)/alloy
ingress: check-context ## Deploy/update the Ingresses and the certificate
	$(KUBECTL) apply -k $(K8S)/ingress

.PHONY: secret
secret: ## Generate k8s/secret.yaml (random admin password) if absent
	if [ ! -f $(K8S)/secret.yaml ]; then \
		pw=$$(LC_ALL=C tr -dc 'A-Za-z0-9' </dev/urandom | head -c 20); \
		sed -e '1s|.*|# Generated by `make secret` -- NOT versioned (see .gitignore).|' \
			-e "s|<CHANGE_ME>|$$pw|" $(K8S)/secret.example.yaml > $(K8S)/secret.yaml; \
		printf "$(C_OK)secret.yaml generated$(C_OFF) -- admin password: $$pw\n"; \
	else \
		printf "secret.yaml already present (use make grafana-password to read it)\n"; \
	fi

##@ Dashboards (declarative, stable UIDs)

.PHONY: dashboards
dashboards: check-context dashboards-check ## Apply dashboards only (no Grafana restart)
	$(KUBECTL) apply -k $(DASHBOARDS)
	printf "$(C_OK)grafana-dashboards ConfigMap updated$(C_OFF) -- visible in Grafana within ~70s\n"
	printf "  (kubelet volume sync ~60s + provisioner poll 10s, no restart).\n"
	printf "  For immediate effect: $(C_WARN)make restart-grafana$(C_OFF)\n"

.PHONY: dashboards-sync
dashboards-sync: ## Regenerate the .json list in dashboards/kustomization.yaml
	cd $(DASHBOARDS) && python3 -c "import glob,re,pathlib; \
p=pathlib.Path('kustomization.yaml'); s=p.read_text(); \
files=sorted(glob.glob('*.json')); \
block=''.join('      - %s\n' % f for f in files); \
s=re.sub(r'(# >>> dashboards.*?>>>\n).*?(      # <<< dashboards <<<)', lambda m: m.group(1)+block+m.group(2), s, flags=re.S); \
p.write_text(s); print('%d dashboard(s) referenced' % len(files))"
	$(MAKE) --no-print-directory dashboards-check

.PHONY: dashboards-check
dashboards-check: ## Validate the JSON files and UID uniqueness/stability
	fail=0; \
	for f in $(DASHBOARDS)/*.json; do \
		jq -e . "$$f" >/dev/null 2>&1 || { printf "$(C_ERR)invalid JSON$(C_OFF): %s\n" "$$f"; fail=1; continue; }; \
		uid=$$(jq -r '.uid // empty' "$$f"); \
		[ -n "$$uid" ] || { printf "$(C_ERR)missing uid$(C_OFF): %s\n" "$$f"; fail=1; }; \
		base=$$(basename "$$f" .json); \
		[ "$$uid" = "$$base" ] || printf "$(C_WARN)filename != uid$(C_OFF): %s (uid=%s)\n" "$$f" "$$uid"; \
		[ "$$(jq -r '.id // "null"' "$$f")" = "null" ] || printf "$(C_WARN)non-null id field$(C_OFF): %s (remove it)\n" "$$f"; \
		grep -q -- "- $$base.json" $(DASHBOARDS)/kustomization.yaml || { printf "$(C_ERR)not referenced in kustomization.yaml$(C_OFF): %s (run make dashboards-sync)\n" "$$f"; fail=1; }; \
	done; \
	dup=$$(jq -r '.uid' $(DASHBOARDS)/*.json | sort | uniq -d); \
	[ -z "$$dup" ] || { printf "$(C_ERR)duplicate UIDs$(C_OFF): %s\n" "$$dup"; fail=1; }; \
	[ $$fail -eq 0 ] && printf "$(C_OK)dashboards OK$(C_OFF) (%s)\n" "$$(ls $(DASHBOARDS)/*.json | wc -l | tr -d ' ')"; \
	exit $$fail

.PHONY: dashboards-list
dashboards-list: ## List the dashboards actually loaded by Grafana
	auth="$$($(KUBECTL) -n $(NS) get secret observability-secrets -o jsonpath='{.data.GF_SECURITY_ADMIN_USER}' | base64 -d):$$($(KUBECTL) -n $(NS) get secret observability-secrets -o jsonpath='{.data.GF_SECURITY_ADMIN_PASSWORD}' | base64 -d)"; \
	$(KUBECTL) -n $(NS) exec deployment/grafana -- curl -s -u "$$auth" 'http://localhost:3000/api/search?type=dash-db' \
		| jq -r '.[] | "\(.uid)\t\(.folderTitle // "General")\t\(.title)"'

##@ Diagnostics

.PHONY: status
status: ## Overview (pods, services, PVCs, ingresses)
	$(KUBECTL) -n $(NS) get pods -o wide
	echo
	$(KUBECTL) -n $(NS) get svc,pvc,ingress

.PHONY: diag
diag: ## Full diagnostics: state, PVCs, endpoints, certificate, events
	printf "$(C_HEAD)== Pods ==$(C_OFF)\n";       $(KUBECTL) -n $(NS) get pods -o wide
	printf "\n$(C_HEAD)== PVCs ==$(C_OFF)\n";      $(KUBECTL) -n $(NS) get pvc
	printf "\n$(C_HEAD)== Endpoints ==$(C_OFF)\n"; $(KUBECTL) -n $(NS) get endpoints
	printf "\n$(C_HEAD)== Ingresses ==$(C_OFF)\n"; $(KUBECTL) -n $(NS) get ingress
	printf "\n$(C_HEAD)== Certificate ==$(C_OFF)\n";$(KUBECTL) -n $(NS) get certificate 2>/dev/null || echo "(cert-manager absent)"
	printf "\n$(C_HEAD)== ConfigMaps ==$(C_OFF)\n";$(KUBECTL) -n $(NS) get cm
	printf "\n$(C_HEAD)== Containers not ready ==$(C_OFF)\n"
	$(KUBECTL) -n $(NS) get pods -o json | jq -r '.items[] | select(.status.containerStatuses != null) | .status.containerStatuses[] | select(.ready == false) | "\(.name)\trestarts=\(.restartCount)\t\(.state | keys[0]): \(.state[keys[0]].reason // "-")"' || true
	printf "\n$(C_HEAD)== LibreChat scrape token ==$(C_OFF)\n"
	$(MAKE) --no-print-directory check-metrics-token || true
	printf "\n$(C_HEAD)== Last 15 events ==$(C_OFF)\n"
	$(KUBECTL) -n $(NS) get events --sort-by=.lastTimestamp | tail -15

.PHONY: smoke
smoke: ## End-to-end checks: component health, datasources, dashboards, targets
	# The Loki and Tempo images ship neither curl nor wget, so probes run from the
	# Grafana pod, which has curl and sees the same Services as the datasources.
	probe() { $(KUBECTL) -n $(NS) exec deployment/grafana -- curl -s --max-time 10 "$$1"; }; \
	auth="$$($(KUBECTL) -n $(NS) get secret observability-secrets -o jsonpath='{.data.GF_SECURITY_ADMIN_USER}' | base64 -d):$$($(KUBECTL) -n $(NS) get secret observability-secrets -o jsonpath='{.data.GF_SECURITY_ADMIN_PASSWORD}' | base64 -d)"; \
	printf "$(C_HEAD)== Component health ==$(C_OFF)\n"; \
	printf "  %-12s %s\n" "prometheus" "$$(probe http://prometheus:9090/-/ready)"; \
	printf "  %-12s %s\n" "loki"       "$$(probe http://loki:3100/ready)"; \
	printf "  %-12s %s\n" "tempo"      "$$(probe http://tempo:3200/ready)"; \
	printf "  %-12s %s\n" "grafana"    "$$(probe http://localhost:3000/api/health | jq -c .)"; \
	printf "  %-12s %s\n" "alloy"      "$$(probe http://alloy:12345/-/ready)"; \
	printf "\n$(C_HEAD)== Grafana datasources ==$(C_OFF)\n"; \
	for uid in prometheus loki tempo; do \
		r=$$($(KUBECTL) -n $(NS) exec deployment/grafana -- curl -s -u "$$auth" -X POST "http://localhost:3000/api/datasources/uid/$$uid/health" | jq -r '"\(.status)  \(.message)"'); \
		printf "  %-12s %s\n" "$$uid" "$$r"; \
	done; \
	printf "\n$(C_HEAD)== Provisioned dashboards ==$(C_OFF)\n"; \
	$(KUBECTL) -n $(NS) exec deployment/grafana -- curl -s -u "$$auth" 'http://localhost:3000/api/search?type=dash-db' \
		| jq -r '.[] | "  \(.uid)  \(.folderTitle // "General")/\(.title)"'; \
	printf "\n$(C_HEAD)== Namespaces seen by Loki ==$(C_OFF)\n"; \
	probe 'http://loki:3100/loki/api/v1/label/namespace/values' | jq -r '"  " + (.data // [] | join(", "))'; \
	printf "\n$(C_HEAD)== Prometheus targets ==$(C_OFF)\n"; \
	probe http://prometheus:9090/api/v1/targets | jq -r '.data.activeTargets | group_by(.health)[] | "  \(.[0].health)\t\(length)"'

.PHONY: targets
targets: ## List Prometheus scrape targets and their state
	$(KUBECTL) -n $(NS) exec deployment/prometheus -- wget -qO- http://localhost:9090/api/v1/targets \
		| jq -r '.data.activeTargets[] | "\(.health)\t\(.labels.job)\t\(.scrapeUrl)\t\(.lastError // "")"' | sort

.PHONY: check-metrics-token
check-metrics-token: ## Check the LibreChat token matches across both namespaces
	a=$$($(KUBECTL) -n $(NS) get secret librechat-metrics -o jsonpath='{.data.token}' 2>/dev/null | base64 -d); \
	b=$$($(KUBECTL) -n librechat get secret librechat-secrets -o jsonpath='{.data.METRICS_SECRET}' 2>/dev/null | base64 -d); \
	if [ -z "$$a" ] || [ -z "$$b" ]; then \
		printf "$(C_WARN)token not found$(C_OFF) (observability/librechat-metrics=%s, librechat/librechat-secrets=%s)\n" "$${a:+present}$${a:-absent}" "$${b:+present}$${b:-absent}"; \
	elif [ "$$a" = "$$b" ]; then \
		printf "$(C_OK)LibreChat token matches$(C_OFF) across observability and librechat\n"; \
	else \
		printf "$(C_ERR)DRIFT$(C_OFF): the two Secrets hold different values.\n"; \
		printf "  The librechat scrape will fall back to 401. Realign both copies.\n"; \
		exit 1; \
	fi

.PHONY: logs-prometheus logs-loki logs-tempo logs-grafana logs-alloy
logs-prometheus: ## Follow Prometheus logs
	$(KUBECTL) -n $(NS) logs -f deployment/prometheus --tail=100
logs-loki: ## Follow Loki logs
	$(KUBECTL) -n $(NS) logs -f deployment/loki --tail=100
logs-tempo: ## Follow Tempo logs
	$(KUBECTL) -n $(NS) logs -f deployment/tempo --tail=100
logs-grafana: ## Follow Grafana logs
	$(KUBECTL) -n $(NS) logs -f deployment/grafana --tail=100
logs-alloy: ## Follow Alloy logs
	$(KUBECTL) -n $(NS) logs -f daemonset/alloy --tail=100

.PHONY: events
events: ## Namespace events, oldest to newest
	$(KUBECTL) -n $(NS) get events --sort-by=.lastTimestamp

.PHONY: top
top: ## Pod CPU/memory usage
	$(KUBECTL) -n $(NS) top pods 2>/dev/null || echo "metrics-server unavailable"

.PHONY: urls
urls: ## Show access URLs
	printf "  Grafana     https://grafana.sttlab.local\n"
	printf "  Prometheus  https://prometheus.sttlab.local\n"
	printf "  Alloy       https://alloy.sttlab.local\n"
	printf "\n  In-cluster: http://prometheus:9090 - http://loki:3100 - http://tempo:3200\n"
	printf "  OTLP      : tempo.$(NS).svc.cluster.local:4317 (gRPC) / :4318 (HTTP)\n"

.PHONY: hosts
hosts: ## Show the required /etc/hosts entries
	printf "# Add to /etc/hosts:\n"
	for h in $(HOSTS); do printf "$(INGRESS_IP)\t%s\n" "$$h"; done

.PHONY: grafana-password
grafana-password: ## Show the Grafana admin credentials
	printf "user     : %s\n" "$$($(KUBECTL) -n $(NS) get secret observability-secrets -o jsonpath='{.data.GF_SECURITY_ADMIN_USER}' | base64 -d)"
	printf "password : %s\n" "$$($(KUBECTL) -n $(NS) get secret observability-secrets -o jsonpath='{.data.GF_SECURITY_ADMIN_PASSWORD}' | base64 -d)"

.PHONY: pf-grafana pf-prometheus pf-loki pf-tempo pf-alloy
pf-grafana: ## Port-forward Grafana on :3000
	$(KUBECTL) -n $(NS) port-forward svc/grafana 3000:3000
pf-prometheus: ## Port-forward Prometheus on :9090
	$(KUBECTL) -n $(NS) port-forward svc/prometheus 9090:9090
pf-loki: ## Port-forward Loki on :3100
	$(KUBECTL) -n $(NS) port-forward svc/loki 3100:3100
pf-tempo: ## Port-forward Tempo on :3200
	$(KUBECTL) -n $(NS) port-forward svc/tempo 3200:3200
pf-alloy: ## Port-forward the Alloy UI on :12345
	$(KUBECTL) -n $(NS) port-forward svc/alloy 12345:12345

##@ Operations

.PHONY: wait
wait: ## Wait for every workload to become ready
	for w in $(WORKLOADS); do \
		printf "  %-24s " "$$w"; \
		$(KUBECTL) -n $(NS) rollout status $$w --timeout=300s 2>&1 | tail -1; \
	done

.PHONY: restart
restart: ## Restart every workload
	$(KUBECTL) -n $(NS) rollout restart $(WORKLOADS)
	$(MAKE) --no-print-directory wait

.PHONY: restart-prometheus restart-loki restart-tempo restart-grafana restart-alloy
restart-prometheus: ## Restart Prometheus
	$(KUBECTL) -n $(NS) rollout restart deployment/prometheus && $(KUBECTL) -n $(NS) rollout status deployment/prometheus
restart-loki: ## Restart Loki
	$(KUBECTL) -n $(NS) rollout restart deployment/loki && $(KUBECTL) -n $(NS) rollout status deployment/loki
restart-tempo: ## Restart Tempo
	$(KUBECTL) -n $(NS) rollout restart deployment/tempo && $(KUBECTL) -n $(NS) rollout status deployment/tempo
restart-grafana: ## Restart Grafana (dashboards picked up immediately)
	$(KUBECTL) -n $(NS) rollout restart deployment/grafana && $(KUBECTL) -n $(NS) rollout status deployment/grafana
restart-alloy: ## Restart Alloy
	$(KUBECTL) -n $(NS) rollout restart daemonset/alloy && $(KUBECTL) -n $(NS) rollout status daemonset/alloy

.PHONY: reload-prometheus
reload-prometheus: ## Hot-reload the Prometheus config (no restart)
	$(KUBECTL) -n $(NS) exec deployment/prometheus -- wget -q --post-data='' -O- http://localhost:9090/-/reload \
		&& printf "$(C_OK)Prometheus configuration reloaded$(C_OFF)\n"

##@ Persistence

.PHONY: pvc
pvc: ## PVC state and the volumes behind them
	$(KUBECTL) -n $(NS) get pvc -o custom-columns='NAME:.metadata.name,STATUS:.status.phase,CAPACITY:.status.capacity.storage,SC:.spec.storageClassName,VOLUME:.spec.volumeName'
	echo
	$(KUBECTL) -n $(NS) get pods -o json | jq -r '.items[] | . as $$p | (.spec.volumes[]? | select(.persistentVolumeClaim) | "\($$p.metadata.name)\t-> \(.persistentVolumeClaim.claimName)")'

.PHONY: flush-pvc
flush-pvc: ## DESTRUCTIVE: wipe all persisted data (CONFIRM=yes)
	$(call confirm,Wipe ALL data (metrics, logs, traces, Grafana))
	for c in $(STATEFUL); do $(MAKE) --no-print-directory _flush COMP=$$c; done
	$(MAKE) --no-print-directory wait

.PHONY: flush-prometheus flush-loki flush-tempo flush-grafana
flush-prometheus: ## DESTRUCTIVE: wipe the Prometheus TSDB (CONFIRM=yes)
	$(call confirm,Wipe the Prometheus TSDB)
	$(MAKE) --no-print-directory _flush COMP=prometheus
flush-loki: ## DESTRUCTIVE: wipe the Loki logs (CONFIRM=yes)
	$(call confirm,Wipe the Loki logs)
	$(MAKE) --no-print-directory _flush COMP=loki
flush-tempo: ## DESTRUCTIVE: wipe the Tempo traces (CONFIRM=yes)
	$(call confirm,Wipe the Tempo traces)
	$(MAKE) --no-print-directory _flush COMP=tempo
flush-grafana: ## DESTRUCTIVE: wipe the Grafana database (CONFIRM=yes)
	$(call confirm,Wipe the Grafana database (users and prefs; dashboards are reprovisioned))
	$(MAKE) --no-print-directory _flush COMP=grafana

# Internal target: scale down -> delete PVC -> recreate -> scale up.
.PHONY: _flush
_flush:
	printf "$(C_WARN)flushing %s$(C_OFF)\n" "$(COMP)"
	$(KUBECTL) -n $(NS) scale deployment/$(COMP) --replicas=0
	$(KUBECTL) -n $(NS) wait --for=delete pod -l app=$(COMP) --timeout=120s 2>/dev/null || true
	$(KUBECTL) -n $(NS) delete pvc $(COMP)-data --ignore-not-found --wait=true
	$(KUBECTL) apply -k $(K8S)/$(COMP)
	$(KUBECTL) -n $(NS) scale deployment/$(COMP) --replicas=1

##@ Validation and teardown

.PHONY: build
build: ## Render the generated manifests to stdout
	kustomize build $(K8S)

.PHONY: diff
diff: check-context ## Diff the manifests against the cluster state
	$(KUBECTL) diff -k $(K8S) || true

.PHONY: validate
validate: dashboards-check ## Validate the manifests (build + server dry-run)
	kustomize build $(K8S) >/dev/null && printf "$(C_OK)kustomize build OK$(C_OFF)\n"
	$(KUBECTL) apply -k $(K8S) --dry-run=server >/dev/null && printf "$(C_OK)server dry-run OK$(C_OFF)\n"

.PHONY: undeploy
undeploy: ## DESTRUCTIVE: delete the stack, PVCs kept (CONFIRM=yes)
	$(call confirm,Delete the stack (PVCs are KEPT))
	$(KUBECTL) delete -k $(K8S) --ignore-not-found

.PHONY: destroy
destroy: ## DESTRUCTIVE: delete the stack AND the PVCs (CONFIRM=yes)
	$(call confirm,Delete the stack AND all data)
	$(KUBECTL) delete -k $(K8S) --ignore-not-found
	$(KUBECTL) -n $(NS) delete pvc --all --ignore-not-found

.PHONY: check-context
check-context:
	cur=$$(kubectl config current-context); \
	if [ "$$cur" != "$(CTX)" ]; then \
		printf "$(C_ERR)current context '%s' != '$(CTX)'$(C_OFF) (commands explicitly target $(CTX))\n" "$$cur"; \
	fi

# Guard rail for destructive targets: requires CONFIRM=yes
define confirm
@if [ "$(CONFIRM)" != "yes" ]; then \
	printf "$(C_ERR)Destructive operation$(C_OFF): $(1)\n"; \
	printf "Re-run with $(C_WARN)CONFIRM=yes$(C_OFF): make $(C_WARN)$@ CONFIRM=yes$(C_OFF)\n"; \
	exit 1; \
fi
endef