diff --git a/.claude/rules/tracing-setup.md b/.claude/rules/tracing-setup.md new file mode 100644 index 00000000..0bc9cc79 --- /dev/null +++ b/.claude/rules/tracing-setup.md @@ -0,0 +1,38 @@ +## Accessing Jaeger Tracing + +**Local Setup:** Jaeger is deployed automatically and configured for both control and data plane tracing. + +1. **Control plane tracing** (kuadrant-operator): + - OTEL env vars automatically configured when `INSTALL_TRACING=true` (default) + - Operator sends traces to `jaeger-collector.tools.svc.cluster.local:4317` + - Trace reconciliation loops, policy processing, and webhook calls + +2. **Data plane tracing** (gateway/envoy): + - Configured in Kuadrant CR: `spec.observability.tracing.defaultEndpoint` + - Gateway sends request traces to same Jaeger collector + - Trace HTTP requests, rate limit checks, and auth decisions + +3. **Access Jaeger UI:** + ```bash + kubectl port-forward -n tools svc/jaeger-query 16686:80 + # Open http://localhost:16686 + ``` + +4. **Run tracing tests:** + ```bash + # Control plane tracing tests (40 tests) + make testsuite/tests/singlecluster/tracing/control_plane/ + + # Data plane tracing tests (10 tests) + make testsuite/tests/singlecluster/tracing/data_plane_tracing/ + ``` + +**Disable tracing:** +```bash +INSTALL_TRACING=false make local-setup +``` + +**View traces:** +- Service: `kuadrant-operator` for control plane traces +- Service: Gateway name for data plane traces +- Filter by operation, tags, or duration diff --git a/README.md b/README.md index 8c6c542e..6e97703b 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,29 @@ This will: make local-cleanup # Delete the Kind cluster ``` +### Tracing (Jaeger) + +Jaeger is deployed automatically by `make local-setup` and configured for both control plane and data plane tracing (when `INSTALL_TRACING=true`, which is the default). + +- **Control plane**: the kuadrant-operator sends traces via OTLP/gRPC to `jaeger-collector.tools.svc.cluster.local:4317` +- **Data plane**: the gateway (Istio/Envoy) sends request-level traces to the same collector + +To access the Jaeger UI: +```bash +kubectl port-forward -n tools svc/jaeger-query 16686:80 +# Open http://localhost:16686 +``` + +To run tracing tests: +```bash +make testsuite/tests/singlecluster/tracing/ +``` + +To disable tracing during setup: +```bash +INSTALL_TRACING=false make local-setup +``` + ## Configuration The Kuadrant testsuite uses [Dynaconf](https://www.dynaconf.com/) for configuration. diff --git a/make/istio.mk b/make/istio.mk index 81a6d14d..437c3e03 100644 --- a/make/istio.mk +++ b/make/istio.mk @@ -28,3 +28,53 @@ istio-install: ## Install Istio via SAIL operator ' version: $(ISTIO_VERSION)' \ | kubectl apply -f - @echo "Istio $(ISTIO_VERSION) installed via SAIL" + +.PHONY: configure-istio-tracing +configure-istio-tracing: ## Configure Istio for distributed tracing + @echo "Configuring Istio for tracing with Jaeger..." + @# Patch Istio CR to add tracing extension provider and JSON access logs + @printf '%s' \ + '{"spec":{"values":{"meshConfig":{' \ + '"accessLogFile":"/dev/stdout",' \ + '"accessLogEncoding":"JSON",' \ + '"accessLogFormat":"{' \ + '\"start_time\":\"%START_TIME%\",' \ + '\"method\":\"%REQ(:METHOD)%\",' \ + '\"path\":\"%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%\",' \ + '\"protocol\":\"%PROTOCOL%\",' \ + '\"response_code\":\"%RESPONSE_CODE%\",' \ + '\"response_flags\":\"%RESPONSE_FLAGS%\",' \ + '\"bytes_received\":\"%BYTES_RECEIVED%\",' \ + '\"bytes_sent\":\"%BYTES_SENT%\",' \ + '\"duration\":\"%DURATION%\",' \ + '\"upstream_service_time\":\"%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%\",' \ + '\"x_forwarded_for\":\"%REQ(X-FORWARDED-FOR)%\",' \ + '\"user_agent\":\"%REQ(USER-AGENT)%\",' \ + '\"request_id\":\"%REQ(X-REQUEST-ID)%\",' \ + '\"authority\":\"%REQ(:AUTHORITY)%\",' \ + '\"upstream_host\":\"%UPSTREAM_HOST%\",' \ + '\"upstream_cluster\":\"%UPSTREAM_CLUSTER%\",' \ + '\"route_name\":\"%ROUTE_NAME%\"' \ + '}",' \ + '"enableTracing":true,' \ + '"defaultConfig":{"tracing":{}},' \ + '"extensionProviders":[{"name":"jaeger-otlp","opentelemetry":{' \ + '"port":4317,' \ + '"service":"jaeger-collector.$(TOOLS_NAMESPACE).svc.cluster.local"' \ + '}}]' \ + '}}}}' \ + | kubectl patch istio default -n istio-system --type=merge --patch-file /dev/stdin + @# Create Telemetry resource to enable tracing + @printf '%s\n' \ + 'apiVersion: telemetry.istio.io/v1' \ + 'kind: Telemetry' \ + 'metadata:' \ + ' name: default-telemetry' \ + ' namespace: istio-system' \ + 'spec:' \ + ' tracing:' \ + ' - providers:' \ + ' - name: jaeger-otlp' \ + ' randomSamplingPercentage: 100' \ + | kubectl apply -f - + @echo "Istio tracing configured" diff --git a/make/kuadrant.mk b/make/kuadrant.mk index 7e96182c..13dd3bbc 100644 --- a/make/kuadrant.mk +++ b/make/kuadrant.mk @@ -30,6 +30,15 @@ endif $(MAKE) patch-kuadrant-operator-env @echo "Kuadrant Operator $(KUADRANT_OPERATOR_VERSION) installed from Helm" endif +ifeq ($(INSTALL_TRACING),true) + @echo "Configuring OTEL tracing on limitador-operator..." + @kubectl set env deployment/limitador-operator-controller-manager \ + -n $(KUADRANT_NAMESPACE) \ + OTEL_EXPORTER_OTLP_ENDPOINT=$(JAEGER_COLLECTOR_ENDPOINT) \ + OTEL_EXPORTER_OTLP_INSECURE=true + @kubectl rollout status deployment/limitador-operator-controller-manager \ + -n $(KUADRANT_NAMESPACE) --timeout=$(KUBECTL_TIMEOUT) +endif .PHONY: patch-kuadrant-operator-env patch-kuadrant-operator-env: ## Patch Kuadrant Operator deployment with custom env vars @@ -56,29 +65,26 @@ else endif .PHONY: deploy-kuadrant-cr -deploy-kuadrant-cr: ## Deploy Kuadrant CR +deploy-kuadrant-cr: ## Deploy Kuadrant CR (composed from INSTALL_PROMETHEUS and INSTALL_TRACING flags) @echo "Creating Kuadrant CR..." -ifeq ($(INSTALL_PROMETHEUS),true) - @echo "Enabling observability in Kuadrant CR..." - @printf '%s\n' \ - 'apiVersion: kuadrant.io/v1beta1' \ - 'kind: Kuadrant' \ - 'metadata:' \ - ' name: kuadrant-sample' \ - ' namespace: $(KUADRANT_NAMESPACE)' \ - 'spec:' \ - ' observability:' \ - ' enable: true' \ - | kubectl apply -f - -else - @printf '%s\n' \ - 'apiVersion: kuadrant.io/v1beta1' \ - 'kind: Kuadrant' \ - 'metadata:' \ - ' name: kuadrant-sample' \ - ' namespace: $(KUADRANT_NAMESPACE)' \ - 'spec: {}' \ - | kubectl apply -f - -endif + @{ \ + echo 'apiVersion: kuadrant.io/v1beta1'; \ + echo 'kind: Kuadrant'; \ + echo 'metadata:'; \ + echo ' name: kuadrant-sample'; \ + echo ' namespace: $(KUADRANT_NAMESPACE)'; \ + echo 'spec:'; \ + echo ' observability:'; \ + echo ' enable: $(INSTALL_PROMETHEUS)'; \ + if [ "$(INSTALL_TRACING)" = "true" ]; then \ + echo ' dataPlane:'; \ + echo ' defaultLevels:'; \ + echo ' - debug: "true"'; \ + echo ' httpHeaderIdentifier: x-request-id'; \ + echo ' tracing:'; \ + echo ' defaultEndpoint: "$(JAEGER_COLLECTOR_ENDPOINT)"'; \ + echo ' insecure: true'; \ + fi; \ + } | kubectl apply -f - kubectl wait kuadrant/kuadrant-sample --for=condition=Ready=True -n $(KUADRANT_NAMESPACE) --timeout=$(KUADRANT_CR_TIMEOUT) @echo "Kuadrant CR ready" diff --git a/make/local-setup.mk b/make/local-setup.mk index 32e42040..1de28ccd 100644 --- a/make/local-setup.mk +++ b/make/local-setup.mk @@ -22,9 +22,14 @@ local-setup: ## Complete local environment setup (kind cluster + all dependencie $(MAKE) $(GATEWAYAPI_PROVIDER)-install $(MAKE) create-test-namespaces $(MAKE) apply-additional-manifests + $(MAKE) deploy-testsuite-tools +ifeq ($(INSTALL_TRACING),true) +ifeq ($(GATEWAYAPI_PROVIDER),istio) + $(MAKE) configure-istio-tracing +endif +endif $(MAKE) deploy-kuadrant-operator $(MAKE) deploy-kuadrant-cr - $(MAKE) deploy-testsuite-tools @echo "" @echo "Local environment setup complete!" @echo " Cluster: $(KIND_CLUSTER_NAME)" @@ -36,13 +41,12 @@ ifeq ($(INSTALL_PROMETHEUS),true) [ -z "$$HOST" ] && HOST=$$(kubectl get svc -n $(PROMETHEUS_NAMESPACE) prometheus-kube-prometheus-prometheus \ -o jsonpath='{.status.loadBalancer.ingress[0].hostname}' 2>/dev/null); \ [ -n "$$HOST" ] && echo " Prometheus URL: http://$$HOST:9090" +endif +ifeq ($(INSTALL_TRACING),true) + @echo " Tracing: Enabled" endif @echo "" @echo "Run tests with: make kuadrant" -ifeq ($(INSTALL_PROMETHEUS),true) - @echo "" - @echo "For observability tests: make observability" -endif .PHONY: local-cleanup local-cleanup: ## Delete local kind cluster and stop background processes diff --git a/make/tools.mk b/make/tools.mk index d5e559aa..2a80791d 100644 --- a/make/tools.mk +++ b/make/tools.mk @@ -3,11 +3,12 @@ .PHONY: deploy-testsuite-tools deploy-testsuite-tools: ## Deploy testsuite tools (Keycloak, etc.) - @echo "Deploying testsuite tools..." - kubectl create namespace tools || true + @echo "Deploying testsuite tools to namespace: $(TOOLS_NAMESPACE)" + kubectl create namespace $(TOOLS_NAMESPACE) || true helm repo add kuadrant-olm https://kuadrant.io/helm-charts-olm --force-update helm repo update helm install \ + --namespace $(TOOLS_NAMESPACE) \ --set=tools.keycloak.keycloakProvider=deployment \ --debug \ --wait \ diff --git a/make/vars.mk b/make/vars.mk index 833c8827..40dc9459 100644 --- a/make/vars.mk +++ b/make/vars.mk @@ -41,6 +41,17 @@ KUADRANT_OPERATOR_ENV_VARS ?= AUTH_SERVICE_TIMEOUT=1000ms,RATELIMIT_SERVICE_TIME # Point to a YAML file containing any additional Kubernetes resources ADDITIONAL_MANIFESTS ?= +# Tools namespace (Jaeger, Keycloak, etc.) +TOOLS_NAMESPACE ?= tools + +# Tracing configuration +INSTALL_TRACING ?= true +JAEGER_COLLECTOR_ENDPOINT ?= rpc://jaeger-collector.$(TOOLS_NAMESPACE).svc.cluster.local:4317 + +ifeq ($(INSTALL_TRACING),true) +KUADRANT_OPERATOR_ENV_VARS := $(KUADRANT_OPERATOR_ENV_VARS),OTEL_EXPORTER_OTLP_ENDPOINT=$(JAEGER_COLLECTOR_ENDPOINT),OTEL_EXPORTER_OTLP_INSECURE=true,LOG_LEVEL=debug +endif + # Timeout configurations (in seconds) KUBECTL_TIMEOUT ?= 300s CERT_MANAGER_TIMEOUT ?= 120s