feat: add distributed tracing (Jaeger) to local-setup

.claude/rules/tracing-setup.md

@@ -0,0 +1,38 @@
+## Accessing Jaeger Tracing
+
+**Local Setup:** Jaeger is deployed automatically and configured for both control and data plane tracing.
+
+1. **Control plane tracing** (kuadrant-operator):
+   - OTEL env vars automatically configured when `INSTALL_TRACING=true` (default)
+   - Operator sends traces to `jaeger-collector.tools.svc.cluster.local:4317`
+   - Trace reconciliation loops, policy processing, and webhook calls
+
+2. **Data plane tracing** (gateway/envoy):
+   - Configured in Kuadrant CR: `spec.observability.tracing.defaultEndpoint`
+   - Gateway sends request traces to same Jaeger collector
+   - Trace HTTP requests, rate limit checks, and auth decisions
+
+3. **Access Jaeger UI:**
+   ```bash
+   kubectl port-forward -n tools svc/jaeger-query 16686:80
+   # Open http://localhost:16686
+   ```
+
+4. **Run tracing tests:**
+   ```bash
+   # Control plane tracing tests (40 tests)
+   make testsuite/tests/singlecluster/tracing/control_plane/
+
+   # Data plane tracing tests (10 tests)
+   make testsuite/tests/singlecluster/tracing/data_plane_tracing/
+   ```
+
+**Disable tracing:**
+```bash
+INSTALL_TRACING=false make local-setup
+```
+
+**View traces:**
+- Service: `kuadrant-operator` for control plane traces
+- Service: Gateway name for data plane traces
+- Filter by operation, tags, or duration

README.md

@@ -84,6 +84,29 @@ This will:
 make local-cleanup  # Delete the Kind cluster
 ```
 
+### Tracing (Jaeger)
+
+Jaeger is deployed automatically by `make local-setup` and configured for both control plane and data plane tracing (when `INSTALL_TRACING=true`, which is the default).
+
+- **Control plane**: the kuadrant-operator sends traces via OTLP/gRPC to `jaeger-collector.tools.svc.cluster.local:4317`
+- **Data plane**: the gateway (Istio/Envoy) sends request-level traces to the same collector
+
+To access the Jaeger UI:
+```bash
+kubectl port-forward -n tools svc/jaeger-query 16686:80
+# Open http://localhost:16686
+```
+
+To run tracing tests:
+```bash
+make testsuite/tests/singlecluster/tracing/
+```
+
+To disable tracing during setup:
+```bash
+INSTALL_TRACING=false make local-setup
+```
+
 ## Configuration
 
 The Kuadrant testsuite uses [Dynaconf](https://www.dynaconf.com/) for configuration.

make/istio.mk

@@ -28,3 +28,53 @@ istio-install: ## Install Istio via SAIL operator
 		'  version: $(ISTIO_VERSION)' \
 		| kubectl apply -f -
 	@echo "Istio $(ISTIO_VERSION) installed via SAIL"
+
+.PHONY: configure-istio-tracing
+configure-istio-tracing: ## Configure Istio for distributed tracing
+	@echo "Configuring Istio for tracing with Jaeger..."
+	@# Patch Istio CR to add tracing extension provider and JSON access logs
+	@printf '%s' \
+		'{"spec":{"values":{"meshConfig":{' \
+		'"accessLogFile":"/dev/stdout",' \
+		'"accessLogEncoding":"JSON",' \
+		'"accessLogFormat":"{' \
+		'\"start_time\":\"%START_TIME%\",' \
+		'\"method\":\"%REQ(:METHOD)%\",' \
+		'\"path\":\"%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%\",' \
+		'\"protocol\":\"%PROTOCOL%\",' \
+		'\"response_code\":\"%RESPONSE_CODE%\",' \
+		'\"response_flags\":\"%RESPONSE_FLAGS%\",' \
+		'\"bytes_received\":\"%BYTES_RECEIVED%\",' \
+		'\"bytes_sent\":\"%BYTES_SENT%\",' \
+		'\"duration\":\"%DURATION%\",' \
+		'\"upstream_service_time\":\"%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%\",' \
+		'\"x_forwarded_for\":\"%REQ(X-FORWARDED-FOR)%\",' \
+		'\"user_agent\":\"%REQ(USER-AGENT)%\",' \
+		'\"request_id\":\"%REQ(X-REQUEST-ID)%\",' \
+		'\"authority\":\"%REQ(:AUTHORITY)%\",' \
+		'\"upstream_host\":\"%UPSTREAM_HOST%\",' \
+		'\"upstream_cluster\":\"%UPSTREAM_CLUSTER%\",' \
+		'\"route_name\":\"%ROUTE_NAME%\"' \
+		'}",' \
+		'"enableTracing":true,' \
+		'"defaultConfig":{"tracing":{}},' \
+		'"extensionProviders":[{"name":"jaeger-otlp","opentelemetry":{' \
+		'"port":4317,' \
+		'"service":"jaeger-collector.$(TOOLS_NAMESPACE).svc.cluster.local"' \
+		'}}]' \
+		'}}}}' \
+		| kubectl patch istio default -n istio-system --type=merge --patch-file /dev/stdin
+	@# Create Telemetry resource to enable tracing
+	@printf '%s\n' \
+		'apiVersion: telemetry.istio.io/v1' \
+		'kind: Telemetry' \
+		'metadata:' \
+		'  name: default-telemetry' \
+		'  namespace: istio-system' \
+		'spec:' \
+		'  tracing:' \
+		'  - providers:' \
+		'    - name: jaeger-otlp' \
+		'    randomSamplingPercentage: 100' \
+		| kubectl apply -f -
+	@echo "Istio tracing configured"

make/kuadrant.mk

@@ -30,6 +30,15 @@ endif
 	$(MAKE) patch-kuadrant-operator-env
 	@echo "Kuadrant Operator $(KUADRANT_OPERATOR_VERSION) installed from Helm"
 endif
+ifeq ($(INSTALL_TRACING),true)
+	@echo "Configuring OTEL tracing on limitador-operator..."
+	@kubectl set env deployment/limitador-operator-controller-manager \
+		-n $(KUADRANT_NAMESPACE) \
+		OTEL_EXPORTER_OTLP_ENDPOINT=$(JAEGER_COLLECTOR_ENDPOINT) \
+		OTEL_EXPORTER_OTLP_INSECURE=true
+	@kubectl rollout status deployment/limitador-operator-controller-manager \
+		-n $(KUADRANT_NAMESPACE) --timeout=$(KUBECTL_TIMEOUT)
+endif
 
 .PHONY: patch-kuadrant-operator-env
 patch-kuadrant-operator-env: ## Patch Kuadrant Operator deployment with custom env vars
@@ -56,29 +65,26 @@ else
 endif
 
 .PHONY: deploy-kuadrant-cr
-deploy-kuadrant-cr: ## Deploy Kuadrant CR
+deploy-kuadrant-cr: ## Deploy Kuadrant CR (composed from INSTALL_PROMETHEUS and INSTALL_TRACING flags)
 	@echo "Creating Kuadrant CR..."
-ifeq ($(INSTALL_PROMETHEUS),true)
-	@echo "Enabling observability in Kuadrant CR..."
-	@printf '%s\n' \
-		'apiVersion: kuadrant.io/v1beta1' \
-		'kind: Kuadrant' \
-		'metadata:' \
-		'  name: kuadrant-sample' \
-		'  namespace: $(KUADRANT_NAMESPACE)' \
-		'spec:' \
-		'  observability:' \
-		'    enable: true' \
-		| kubectl apply -f -
-else
-	@printf '%s\n' \
-		'apiVersion: kuadrant.io/v1beta1' \
-		'kind: Kuadrant' \
-		'metadata:' \
-		'  name: kuadrant-sample' \
-		'  namespace: $(KUADRANT_NAMESPACE)' \
-		'spec: {}' \
-		| kubectl apply -f -
-endif
+	@{ \
+		echo 'apiVersion: kuadrant.io/v1beta1'; \
+		echo 'kind: Kuadrant'; \
+		echo 'metadata:'; \
+		echo '  name: kuadrant-sample'; \
+		echo '  namespace: $(KUADRANT_NAMESPACE)'; \
+		echo 'spec:'; \
+		echo '  observability:'; \
+		echo '    enable: $(INSTALL_PROMETHEUS)'; \
+		if [ "$(INSTALL_TRACING)" = "true" ]; then \
+			echo '    dataPlane:'; \
+			echo '      defaultLevels:'; \
+			echo '      - debug: "true"'; \
+			echo '      httpHeaderIdentifier: x-request-id'; \
+			echo '    tracing:'; \
+			echo '      defaultEndpoint: "$(JAEGER_COLLECTOR_ENDPOINT)"'; \
+			echo '      insecure: true'; \
+		fi; \
+	} | kubectl apply -f -
 	kubectl wait kuadrant/kuadrant-sample --for=condition=Ready=True -n $(KUADRANT_NAMESPACE) --timeout=$(KUADRANT_CR_TIMEOUT)
 	@echo "Kuadrant CR ready"

make/local-setup.mk

@@ -22,9 +22,14 @@ local-setup: ## Complete local environment setup (kind cluster + all dependencie
 	$(MAKE) $(GATEWAYAPI_PROVIDER)-install
 	$(MAKE) create-test-namespaces
 	$(MAKE) apply-additional-manifests
+	$(MAKE) deploy-testsuite-tools
+ifeq ($(INSTALL_TRACING),true)
+ifeq ($(GATEWAYAPI_PROVIDER),istio)
+	$(MAKE) configure-istio-tracing
+endif
+endif
 	$(MAKE) deploy-kuadrant-operator
 	$(MAKE) deploy-kuadrant-cr
-	$(MAKE) deploy-testsuite-tools
 	@echo ""
 	@echo "Local environment setup complete!"
 	@echo "   Cluster: $(KIND_CLUSTER_NAME)"
@@ -36,13 +41,12 @@ ifeq ($(INSTALL_PROMETHEUS),true)
 	[ -z "$$HOST" ] && HOST=$$(kubectl get svc -n $(PROMETHEUS_NAMESPACE) prometheus-kube-prometheus-prometheus \
 		-o jsonpath='{.status.loadBalancer.ingress[0].hostname}' 2>/dev/null); \
 	[ -n "$$HOST" ] && echo "   Prometheus URL: http://$$HOST:9090"
+endif
+ifeq ($(INSTALL_TRACING),true)
+	@echo "   Tracing: Enabled"
 endif
 	@echo ""
 	@echo "Run tests with: make kuadrant"
-ifeq ($(INSTALL_PROMETHEUS),true)
-	@echo ""
-	@echo "For observability tests: make observability"
-endif
 
 .PHONY: local-cleanup
 local-cleanup: ## Delete local kind cluster and stop background processes

make/tools.mk

@@ -3,11 +3,12 @@
 
 .PHONY: deploy-testsuite-tools
 deploy-testsuite-tools: ## Deploy testsuite tools (Keycloak, etc.)
-	@echo "Deploying testsuite tools..."
-	kubectl create namespace tools || true
+	@echo "Deploying testsuite tools to namespace: $(TOOLS_NAMESPACE)"
+	kubectl create namespace $(TOOLS_NAMESPACE) || true
 	helm repo add kuadrant-olm https://kuadrant.io/helm-charts-olm --force-update
 	helm repo update
 	helm install \
+		--namespace $(TOOLS_NAMESPACE) \
 		--set=tools.keycloak.keycloakProvider=deployment \
 		--debug \
 		--wait \

make/vars.mk

@@ -41,6 +41,17 @@ KUADRANT_OPERATOR_ENV_VARS ?= AUTH_SERVICE_TIMEOUT=1000ms,RATELIMIT_SERVICE_TIME
 # Point to a YAML file containing any additional Kubernetes resources
 ADDITIONAL_MANIFESTS ?=
 
+# Tools namespace (Jaeger, Keycloak, etc.)
+TOOLS_NAMESPACE ?= tools
+
+# Tracing configuration
+INSTALL_TRACING ?= true
+JAEGER_COLLECTOR_ENDPOINT ?= rpc://jaeger-collector.$(TOOLS_NAMESPACE).svc.cluster.local:4317
+
+ifeq ($(INSTALL_TRACING),true)
+KUADRANT_OPERATOR_ENV_VARS := $(KUADRANT_OPERATOR_ENV_VARS),OTEL_EXPORTER_OTLP_ENDPOINT=$(JAEGER_COLLECTOR_ENDPOINT),OTEL_EXPORTER_OTLP_INSECURE=true,LOG_LEVEL=debug
+endif
+
 # Timeout configurations (in seconds)
 KUBECTL_TIMEOUT ?= 300s
 CERT_MANAGER_TIMEOUT ?= 120s