From dd8fb422d54dcb87557f5da81c83d644eb9a173d Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Mon, 6 Apr 2026 16:26:06 +0200 Subject: [PATCH 1/9] feat: add Jaeger tracing to local-setup Signed-off-by: Alexander Cristurean --- CLAUDE.md | 39 ++++++++++++++++++++++++++++++++++ config/settings.local.yaml.tpl | 6 +++--- make/istio.mk | 20 +++++++++++++++++ make/kuadrant.mk | 18 ++++++++++++++++ make/local-setup.mk | 11 +++++++++- make/tools.mk | 5 +++-- make/vars.mk | 7 ++++++ 7 files changed, 100 insertions(+), 6 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 42cb022a5..8e21115ec 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -71,6 +71,45 @@ make mypy # Run type checking make clean # Delete all test-created resources (uses USER env var) ``` +### Accessing Jaeger Tracing + +**Local Setup:** Jaeger is deployed automatically and configured for both control and data plane tracing. + +1. **Control plane tracing** (kuadrant-operator): + - OTEL env vars automatically configured when `INSTALL_TRACING=true` (default) + - Operator sends traces to `jaeger-collector.tools.svc.cluster.local:4318` + - Trace reconciliation loops, policy processing, and webhook calls + +2. **Data plane tracing** (gateway/envoy): + - Configured in Kuadrant CR: `spec.observability.tracing.defaultEndpoint` + - Gateway sends request traces to same Jaeger collector + - Trace HTTP requests, rate limit checks, and auth decisions + +3. **Access Jaeger UI:** + ```bash + kubectl port-forward -n tools svc/jaeger-query 16686:80 + # Open http://localhost:16686 + ``` + +4. **Run tracing tests:** + ```bash + # Control plane tracing tests (40 tests) + make testsuite/tests/singlecluster/tracing/control_plane/ + + # Data plane tracing tests (10 tests) + make testsuite/tests/singlecluster/tracing/data_plane_tracing/ + ``` + +**Disable tracing:** +```bash +INSTALL_TRACING=false make local-setup +``` + +**View traces:** +- Service: `kuadrant-operator` for control plane traces +- Service: Gateway name for data plane traces +- Filter by operation, tags, or duration + ## Pull Request Guidelines For PR title format and commit conventions, see `.claude/commands/pr-description.md` or use the `/pr-description` command. diff --git a/config/settings.local.yaml.tpl b/config/settings.local.yaml.tpl index 3f8511c94..0fec9b0eb 100644 --- a/config/settings.local.yaml.tpl +++ b/config/settings.local.yaml.tpl @@ -27,9 +27,9 @@ # url: "MOCKSERVER_URL" # image: "MOCKSERVER_IMAGE" # Image to be used for self-deployed Mockserver # tracing: -# backend: "jaeger" # Tracing backend -# collector_url: "rpc://jaeger-collector.com:4317" # Tracing collector URL (may be internal) -# query_url: "http://jaeger-query.com" # Tracing query URL +# backend: "jaeger" # Tracing backend (jaeger or tempo) +# collector_url: "rpc://jaeger-collector.tools.svc.cluster.local:4317" # Collector endpoint (internal cluster DNS for Kind) +# query_url: "http://jaeger-query.tools.svc.cluster.local:80" # Query UI endpoint (internal DNS or LoadBalancer IP) # cfssl: "cfssl" # Path to the CFSSL library for TLS tests # service_protection: # system_project: "kuadrant-system" # Namespace where Kuadrant resource resides diff --git a/make/istio.mk b/make/istio.mk index 81a6d14db..1015f766a 100644 --- a/make/istio.mk +++ b/make/istio.mk @@ -28,3 +28,23 @@ istio-install: ## Install Istio via SAIL operator ' version: $(ISTIO_VERSION)' \ | kubectl apply -f - @echo "Istio $(ISTIO_VERSION) installed via SAIL" + +.PHONY: configure-istio-tracing +configure-istio-tracing: ## Configure Istio for distributed tracing + @echo "Configuring Istio for tracing with Jaeger..." + @# Patch Istio CR to add tracing extension provider and JSON access logs + @kubectl patch istio default -n istio-system --type=merge -p '{"spec":{"values":{"meshConfig":{"accessLogFile":"/dev/stdout","accessLogEncoding":"JSON","accessLogFormat":"{\"start_time\":\"%START_TIME%\",\"method\":\"%REQ(:METHOD)%\",\"path\":\"%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%\",\"protocol\":\"%PROTOCOL%\",\"response_code\":\"%RESPONSE_CODE%\",\"response_flags\":\"%RESPONSE_FLAGS%\",\"bytes_received\":\"%BYTES_RECEIVED%\",\"bytes_sent\":\"%BYTES_SENT%\",\"duration\":\"%DURATION%\",\"upstream_service_time\":\"%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%\",\"x_forwarded_for\":\"%REQ(X-FORWARDED-FOR)%\",\"user_agent\":\"%REQ(USER-AGENT)%\",\"request_id\":\"%REQ(X-REQUEST-ID)%\",\"authority\":\"%REQ(:AUTHORITY)%\",\"upstream_host\":\"%UPSTREAM_HOST%\",\"upstream_cluster\":\"%UPSTREAM_CLUSTER%\",\"route_name\":\"%ROUTE_NAME%\"}","enableTracing":true,"defaultConfig":{"tracing":{}},"extensionProviders":[{"name":"jaeger-otlp","opentelemetry":{"port":4317,"service":"jaeger-collector.$(TOOLS_NAMESPACE).svc.cluster.local"}}]}}}}' + @# Create Telemetry resource to enable tracing + @printf '%s\n' \ + 'apiVersion: telemetry.istio.io/v1' \ + 'kind: Telemetry' \ + 'metadata:' \ + ' name: default-telemetry' \ + ' namespace: istio-system' \ + 'spec:' \ + ' tracing:' \ + ' - providers:' \ + ' - name: jaeger-otlp' \ + ' randomSamplingPercentage: 100' \ + | kubectl apply -f - + @echo "Istio tracing configured" diff --git a/make/kuadrant.mk b/make/kuadrant.mk index 1c6199c19..c81f03729 100644 --- a/make/kuadrant.mk +++ b/make/kuadrant.mk @@ -68,3 +68,21 @@ deploy-kuadrant-cr: ## Deploy Kuadrant CR | kubectl apply -f - kubectl wait kuadrant/kuadrant-sample --for=condition=Ready=True -n $(KUADRANT_NAMESPACE) --timeout=$(KUADRANT_CR_TIMEOUT) @echo "Kuadrant CR ready" + +.PHONY: configure-kuadrant-tracing-operator +configure-kuadrant-tracing-operator: ## Configure OTEL env vars on kuadrant-operator (control plane tracing) + @echo "Configuring OTEL environment variables on kuadrant-operator..." + @kubectl set env deployment/kuadrant-operator-controller-manager \ + -n $(KUADRANT_NAMESPACE) \ + OTEL_EXPORTER_OTLP_ENDPOINT=$(JAEGER_COLLECTOR_ENDPOINT) \ + OTEL_EXPORTER_OTLP_INSECURE=true \ + LOG_LEVEL=debug + @kubectl rollout status deployment/kuadrant-operator-controller-manager \ + -n $(KUADRANT_NAMESPACE) --timeout=$(KUBECTL_TIMEOUT) + @echo "Control plane tracing configured on operator" + +.PHONY: configure-kuadrant-tracing-cr +configure-kuadrant-tracing-cr: ## Configure tracing in Kuadrant CR (data plane tracing) + @echo "Configuring observability (tracing + data plane) in Kuadrant CR..." + @kubectl patch kuadrant kuadrant-sample -n $(KUADRANT_NAMESPACE) --type=merge -p '{"spec":{"observability":{"enable":true,"dataPlane":{"defaultLevels":[{"debug":"true"}],"httpHeaderIdentifier":"x-request-id"},"tracing":{"defaultEndpoint":"$(JAEGER_COLLECTOR_ENDPOINT)","insecure":true}}}}' + @echo "Data plane tracing configured in Kuadrant CR" diff --git a/make/local-setup.mk b/make/local-setup.mk index 69d721d5f..96e4f6daf 100644 --- a/make/local-setup.mk +++ b/make/local-setup.mk @@ -22,9 +22,18 @@ local-setup: ## Complete local environment setup (kind cluster + all dependencie $(MAKE) $(GATEWAYAPI_PROVIDER)-install $(MAKE) create-test-namespaces $(MAKE) apply-additional-manifests + $(MAKE) deploy-testsuite-tools +ifeq ($(INSTALL_TRACING),true) +ifeq ($(GATEWAYAPI_PROVIDER),istio) + $(MAKE) configure-istio-tracing +endif +endif $(MAKE) deploy-kuadrant-operator $(MAKE) deploy-kuadrant-cr - $(MAKE) deploy-testsuite-tools +ifeq ($(INSTALL_TRACING),true) + $(MAKE) configure-kuadrant-tracing-operator + $(MAKE) configure-kuadrant-tracing-cr +endif @echo "" @echo "Local environment setup complete!" @echo " Cluster: $(KIND_CLUSTER_NAME)" diff --git a/make/tools.mk b/make/tools.mk index d5e559aa7..2a80791d8 100644 --- a/make/tools.mk +++ b/make/tools.mk @@ -3,11 +3,12 @@ .PHONY: deploy-testsuite-tools deploy-testsuite-tools: ## Deploy testsuite tools (Keycloak, etc.) - @echo "Deploying testsuite tools..." - kubectl create namespace tools || true + @echo "Deploying testsuite tools to namespace: $(TOOLS_NAMESPACE)" + kubectl create namespace $(TOOLS_NAMESPACE) || true helm repo add kuadrant-olm https://kuadrant.io/helm-charts-olm --force-update helm repo update helm install \ + --namespace $(TOOLS_NAMESPACE) \ --set=tools.keycloak.keycloakProvider=deployment \ --debug \ --wait \ diff --git a/make/vars.mk b/make/vars.mk index cb42f169c..280de8eef 100644 --- a/make/vars.mk +++ b/make/vars.mk @@ -36,6 +36,13 @@ KUADRANT_OPERATOR_ENV_VARS ?= AUTH_SERVICE_TIMEOUT=1000ms,RATELIMIT_SERVICE_TIME # Point to a YAML file containing any additional Kubernetes resources ADDITIONAL_MANIFESTS ?= +# Tools namespace (Jaeger, Keycloak, etc.) +TOOLS_NAMESPACE ?= tools + +# Tracing configuration +INSTALL_TRACING ?= true +JAEGER_COLLECTOR_ENDPOINT ?= rpc://jaeger-collector.$(TOOLS_NAMESPACE).svc.cluster.local:4317 + # Timeout configurations (in seconds) KUBECTL_TIMEOUT ?= 300s CERT_MANAGER_TIMEOUT ?= 120s From 7cd2f4464dcccc9ba46eb564d073432b03ef4f0f Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Tue, 7 Apr 2026 08:34:05 +0200 Subject: [PATCH 2/9] fix: cosmetic changes. Signed-off-by: Alexander Cristurean --- config/settings.local.yaml.tpl | 2 -- 1 file changed, 2 deletions(-) diff --git a/config/settings.local.yaml.tpl b/config/settings.local.yaml.tpl index 0fec9b0eb..9d3c61d68 100644 --- a/config/settings.local.yaml.tpl +++ b/config/settings.local.yaml.tpl @@ -27,8 +27,6 @@ # url: "MOCKSERVER_URL" # image: "MOCKSERVER_IMAGE" # Image to be used for self-deployed Mockserver # tracing: -# backend: "jaeger" # Tracing backend (jaeger or tempo) -# collector_url: "rpc://jaeger-collector.tools.svc.cluster.local:4317" # Collector endpoint (internal cluster DNS for Kind) # query_url: "http://jaeger-query.tools.svc.cluster.local:80" # Query UI endpoint (internal DNS or LoadBalancer IP) # cfssl: "cfssl" # Path to the CFSSL library for TLS tests # service_protection: From 0b3d881e2f31a15e22c52b5242976cfdfa81260c Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Tue, 7 Apr 2026 10:26:43 +0200 Subject: [PATCH 3/9] fix: moved to http port. Signed-off-by: Alexander Cristurean --- make/istio.mk | 2 +- make/vars.mk | 2 +- testsuite/config/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/make/istio.mk b/make/istio.mk index 1015f766a..cf2d859c5 100644 --- a/make/istio.mk +++ b/make/istio.mk @@ -33,7 +33,7 @@ istio-install: ## Install Istio via SAIL operator configure-istio-tracing: ## Configure Istio for distributed tracing @echo "Configuring Istio for tracing with Jaeger..." @# Patch Istio CR to add tracing extension provider and JSON access logs - @kubectl patch istio default -n istio-system --type=merge -p '{"spec":{"values":{"meshConfig":{"accessLogFile":"/dev/stdout","accessLogEncoding":"JSON","accessLogFormat":"{\"start_time\":\"%START_TIME%\",\"method\":\"%REQ(:METHOD)%\",\"path\":\"%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%\",\"protocol\":\"%PROTOCOL%\",\"response_code\":\"%RESPONSE_CODE%\",\"response_flags\":\"%RESPONSE_FLAGS%\",\"bytes_received\":\"%BYTES_RECEIVED%\",\"bytes_sent\":\"%BYTES_SENT%\",\"duration\":\"%DURATION%\",\"upstream_service_time\":\"%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%\",\"x_forwarded_for\":\"%REQ(X-FORWARDED-FOR)%\",\"user_agent\":\"%REQ(USER-AGENT)%\",\"request_id\":\"%REQ(X-REQUEST-ID)%\",\"authority\":\"%REQ(:AUTHORITY)%\",\"upstream_host\":\"%UPSTREAM_HOST%\",\"upstream_cluster\":\"%UPSTREAM_CLUSTER%\",\"route_name\":\"%ROUTE_NAME%\"}","enableTracing":true,"defaultConfig":{"tracing":{}},"extensionProviders":[{"name":"jaeger-otlp","opentelemetry":{"port":4317,"service":"jaeger-collector.$(TOOLS_NAMESPACE).svc.cluster.local"}}]}}}}' + @kubectl patch istio default -n istio-system --type=merge -p '{"spec":{"values":{"meshConfig":{"accessLogFile":"/dev/stdout","accessLogEncoding":"JSON","accessLogFormat":"{\"start_time\":\"%START_TIME%\",\"method\":\"%REQ(:METHOD)%\",\"path\":\"%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%\",\"protocol\":\"%PROTOCOL%\",\"response_code\":\"%RESPONSE_CODE%\",\"response_flags\":\"%RESPONSE_FLAGS%\",\"bytes_received\":\"%BYTES_RECEIVED%\",\"bytes_sent\":\"%BYTES_SENT%\",\"duration\":\"%DURATION%\",\"upstream_service_time\":\"%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%\",\"x_forwarded_for\":\"%REQ(X-FORWARDED-FOR)%\",\"user_agent\":\"%REQ(USER-AGENT)%\",\"request_id\":\"%REQ(X-REQUEST-ID)%\",\"authority\":\"%REQ(:AUTHORITY)%\",\"upstream_host\":\"%UPSTREAM_HOST%\",\"upstream_cluster\":\"%UPSTREAM_CLUSTER%\",\"route_name\":\"%ROUTE_NAME%\"}","enableTracing":true,"defaultConfig":{"tracing":{}},"extensionProviders":[{"name":"jaeger-otlp","opentelemetry":{"port":4318,"service":"jaeger-collector.$(TOOLS_NAMESPACE).svc.cluster.local"}}]}}}}' @# Create Telemetry resource to enable tracing @printf '%s\n' \ 'apiVersion: telemetry.istio.io/v1' \ diff --git a/make/vars.mk b/make/vars.mk index 280de8eef..9f39fc883 100644 --- a/make/vars.mk +++ b/make/vars.mk @@ -41,7 +41,7 @@ TOOLS_NAMESPACE ?= tools # Tracing configuration INSTALL_TRACING ?= true -JAEGER_COLLECTOR_ENDPOINT ?= rpc://jaeger-collector.$(TOOLS_NAMESPACE).svc.cluster.local:4317 +JAEGER_COLLECTOR_ENDPOINT ?= http://jaeger-collector.$(TOOLS_NAMESPACE).svc.cluster.local:4318 # Timeout configurations (in seconds) KUBECTL_TIMEOUT ?= 300s diff --git a/testsuite/config/__init__.py b/testsuite/config/__init__.py index 5bf589c34..4443619e8 100644 --- a/testsuite/config/__init__.py +++ b/testsuite/config/__init__.py @@ -41,7 +41,7 @@ def __init__(self, name, default, **kwargs) -> None: ), DefaultValueValidator("tracing.backend", default="jaeger", is_in=["jaeger", "tempo"]), DefaultValueValidator( - "tracing.collector_url", default=fetch_service("jaeger-collector", protocol="rpc", port=4317) + "tracing.collector_url", default=fetch_service("jaeger-collector", protocol="http", port=4318) ), DefaultValueValidator("tracing.query_url", default=fetch_service_ip("jaeger-query", protocol="http", port=80)), Validator( From 36c8a11e82f7ae64e2bf46be741e9040723cbd1b Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Tue, 7 Apr 2026 17:43:43 +0200 Subject: [PATCH 4/9] feat: adapted tests for kind environment. Signed-off-by: Alexander Cristurean --- .../test_control_plane_lifecycle.py | 28 +++++++++++++++---- testsuite/tracing/__init__.py | 13 +++++++-- testsuite/tracing/jaeger.py | 20 ++++++++++++- testsuite/tracing/tempo.py | 13 ++++++++- 4 files changed, 65 insertions(+), 9 deletions(-) diff --git a/testsuite/tests/singlecluster/tracing/control_plane/test_control_plane_lifecycle.py b/testsuite/tests/singlecluster/tracing/control_plane/test_control_plane_lifecycle.py index 79546a20a..3005d3978 100644 --- a/testsuite/tests/singlecluster/tracing/control_plane/test_control_plane_lifecycle.py +++ b/testsuite/tests/singlecluster/tracing/control_plane/test_control_plane_lifecycle.py @@ -5,6 +5,8 @@ target changes, and multi-policy scenarios. """ +import time + import pytest from testsuite.gateway.gateway_api.route import HTTPRoute @@ -30,21 +32,30 @@ def updated_authorization(authorization, trace_snapshot_before_update): # pylin """ Authorization policy after update. (trace_snapshot_before_update ensures snapshot taken before update) + Returns tuple of (authorization, update_timestamp_micros) """ + # Capture timestamp right before updating (in microseconds) + update_time = int(time.time() * 1_000_000) + when_post = [Pattern("context.request.http.method", "eq", "POST")] authorization.authorization.add_opa_policy("opa", "allow { false }", when=when_post) authorization.wait_for_ready() - return authorization + return authorization, update_time def test_policy_update_generates_new_reconciliation_trace(updated_authorization, trace_snapshot_before_update, tracing): """ Validate that policy updates generate new reconciliation traces """ + authorization, update_time = updated_authorization snapshot = trace_snapshot_before_update - updated_traces = tracing.get_traces(service="kuadrant-operator", tags={"policy.name": updated_authorization.name()}) + # Query for traces that started after the update timestamp + # The backoff decorator will retry until at least one trace appears + updated_traces = tracing.get_traces( + service="kuadrant-operator", tags={"policy.name": authorization.name()}, start_time=update_time + ) # Find new reconcile spans (spans that weren't in the original snapshot) new_reconcile_spans = [] @@ -58,7 +69,7 @@ def test_policy_update_generates_new_reconciliation_trace(updated_authorization, # Find new policy spans (spans that weren't in the original snapshot) new_policy_spans = [] for trace in updated_traces: - for span in trace.filter_spans(lambda s: s.get_tag("policy.name") == updated_authorization.name()): + for span in trace.filter_spans(lambda s: s.get_tag("policy.name") == authorization.name()): if span.span_id not in snapshot["span_ids"]: new_policy_spans.append(span) @@ -214,20 +225,27 @@ def authorization_with_changed_target( """ Authorization policy with targetRef changed to second_route. (trace_snapshot_before_target_change ensures snapshot taken before change) + Returns tuple of (authorization, change_timestamp_micros) """ + # Capture timestamp right before changing the target (in microseconds) + change_time = int(time.time() * 1_000_000) + authorization.refresh() authorization.model.spec.targetRef = second_route.reference authorization.apply() authorization.wait_for_ready() - return authorization + return authorization, change_time def test_policy_target_change_traced(authorization_with_changed_target, trace_snapshot_before_target_change, tracing): """Validate traces when policy's targetRef changes""" + authorization, change_time = authorization_with_changed_target snapshot = trace_snapshot_before_target_change + # Query for traces that started after the target change timestamp + # The backoff decorator will retry until at least one trace appears updated_traces = tracing.get_traces( - service="kuadrant-operator", tags={"policy.name": authorization_with_changed_target.name()} + service="kuadrant-operator", tags={"policy.name": authorization.name()}, start_time=change_time ) # Find new reconcile spans (spans that weren't in the original snapshot) diff --git a/testsuite/tracing/__init__.py b/testsuite/tracing/__init__.py index 2f1a9c861..5bfe20df0 100644 --- a/testsuite/tracing/__init__.py +++ b/testsuite/tracing/__init__.py @@ -23,6 +23,15 @@ def collector_url(self): """Returns URL for application to deposit traces""" @abc.abstractmethod - def get_traces(self, service: str, tags: Optional[dict[str, str]] = None, min_processes: int = 0) -> list[Any]: + def get_traces( + self, + service: str, + tags: Optional[dict[str, str]] = None, + min_processes: int = 0, + lookback: Optional[str] = None, + start_time: Optional[int] = None, + ) -> list[Any]: """Search traces in tracing client by service name and tags. - If min_processes is set, retries until at least that many service processes are present""" + If min_processes is set, retries until at least that many service processes are present. + If lookback is set, only returns traces within that time window (e.g., "1h", "30m"). + If start_time is set, only returns traces that started after that time (in microseconds).""" diff --git a/testsuite/tracing/jaeger.py b/testsuite/tracing/jaeger.py index eb48c29e1..278455e3a 100644 --- a/testsuite/tracing/jaeger.py +++ b/testsuite/tracing/jaeger.py @@ -33,16 +33,34 @@ def query_url(self): return self._query_url @backoff.on_predicate(backoff.fibo, lambda x: x == [], max_tries=7, jitter=None) - def get_traces(self, service: str, tags: Optional[dict[str, str]] = None, min_processes: int = 0) -> list[Trace]: + def get_traces( + self, + service: str, + tags: Optional[dict[str, str]] = None, + min_processes: int = 0, + lookback: Optional[str] = None, + start_time: Optional[int] = None, + ) -> list[Trace]: """Gets trace from tracing backend Tempo or Jaeger. If min_processes is set, retries until at least that many service processes are present. + Args: + service: Service name to filter traces + tags: Optional tags to filter traces + min_processes: Minimum number of processes required in traces + lookback: Optional lookback duration (e.g., "1h", "30m", "1m") + start_time: Optional start time in microseconds (filters traces that started after this time) + Returns: List of Trace objects """ params = {"service": service} if tags: params["tags"] = json.dumps(tags) + if lookback: + params["lookback"] = lookback + if start_time: + params["start"] = start_time traces_data = self.query.api.traces.get(params=params).json()["data"] if not traces_data: diff --git a/testsuite/tracing/tempo.py b/testsuite/tracing/tempo.py index ff53a3246..97456633b 100644 --- a/testsuite/tracing/tempo.py +++ b/testsuite/tracing/tempo.py @@ -12,12 +12,23 @@ class RemoteTempoClient(JaegerClient): """Client to a Tempo that is deployed remotely""" @backoff.on_predicate(backoff.fibo, lambda x: x == [], max_tries=7, jitter=None) - def get_traces(self, service: str, tags: Optional[dict[str, str]] = None, min_processes: int = 0) -> list[Trace]: + def get_traces( + self, + service: str, + tags: Optional[dict[str, str]] = None, + min_processes: int = 0, + lookback: Optional[str] = None, + start_time: Optional[int] = None, + ) -> list[Trace]: """Gets trace from Tempo tracing backend. If min_processes is set, retries until at least that many service processes are present""" params = {"service.name": service} if tags: params.update(tags) + if lookback: + params["lookback"] = lookback + if start_time: + params["start"] = start_time traces_data = self.query.api.get_traces.get(params=params).json()["traces"] if not traces_data: return [] From cef128b9c69c55b529a11e0ba1e7b494b5161485 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Tue, 7 Apr 2026 17:44:25 +0200 Subject: [PATCH 5/9] feat: added test for effective policies. Signed-off-by: Alexander Cristurean --- .../test_control_plane_sources.py | 249 ++++++++++++++++++ 1 file changed, 249 insertions(+) create mode 100644 testsuite/tests/singlecluster/tracing/control_plane/test_control_plane_sources.py diff --git a/testsuite/tests/singlecluster/tracing/control_plane/test_control_plane_sources.py b/testsuite/tests/singlecluster/tracing/control_plane/test_control_plane_sources.py new file mode 100644 index 000000000..34b220457 --- /dev/null +++ b/testsuite/tests/singlecluster/tracing/control_plane/test_control_plane_sources.py @@ -0,0 +1,249 @@ +""" +Control plane tracing tests for source policies attributes. + +Tests verify that reconciliation spans include 'sources' attributes +linking them to the policies that triggered the reconciliation. +""" + +import time + +import pytest + +from testsuite.kuadrant.policy.authorization.auth_policy import AuthPolicy +from testsuite.kuadrant.policy.rate_limit import Limit, RateLimitPolicy +from testsuite.kubernetes import Selector + +pytestmark = [pytest.mark.observability, pytest.mark.limitador, pytest.mark.authorino, pytest.mark.kuadrant_only] + + +@pytest.fixture(scope="module") +def authconfig_trace(auth_traces, skip_or_fail): + """Find trace with authconfig span that has sources attribute""" + for trace in auth_traces: + spans = trace.filter_spans(lambda s: s.operation_name == "authconfig" and s.has_tag("sources")) + if spans: + return trace + + skip_or_fail("No trace with authconfig span containing 'sources' attribute found") + + +@pytest.fixture(scope="module") +def limitador_trace(rl_traces, skip_or_fail): + """Find trace with span that has sources attribute""" + for trace in rl_traces: + spans = trace.filter_spans(lambda s: s.operation_name == "limits" and s.has_tag("sources")) + if spans: + return trace + + available_operations = list({span.operation_name for trace in rl_traces for span in trace.spans})[:20] + skip_or_fail( + f"No trace with spans containing 'sources' attribute found in rate limit traces. " + f"Available operations: {available_operations}" + ) + + +def test_authconfig_span_attributes(authconfig_trace, authorization): + """ + Validate that authconfig reconciliation spans include sources, name, and namespace attributes. + """ + authconfig_span = authconfig_trace.filter_spans( + lambda s: s.operation_name == "authconfig" and s.has_tag("sources") + )[0] + + sources = authconfig_span.get_tag("sources") + assert sources is not None, "sources attribute is None" + assert len(sources) > 0, "sources list is empty" + + policy_ref = f"authpolicy.kuadrant.io:{authorization.namespace()}/{authorization.name()}" + assert policy_ref in sources, f"AuthPolicy {policy_ref} not found in sources: {sources}" + + assert authconfig_span.has_tag("name"), "authconfig span missing 'name' attribute" + assert authconfig_span.has_tag("namespace"), "authconfig span missing 'namespace' attribute" + + name = authconfig_span.get_tag("name") + namespace = authconfig_span.get_tag("namespace") + assert name is not None and name != "", "authconfig name attribute is empty" + assert namespace is not None and namespace != "", "authconfig namespace attribute is empty" + + +def test_limitador_span_attributes(limitador_trace, rate_limit): + """ + Validate that limitador limits reconciliation spans include sources, name, and namespace attributes. + """ + limitador_span = limitador_trace.filter_spans(lambda s: s.has_tag("sources"))[0] + + sources = limitador_span.get_tag("sources") + assert sources is not None, "sources attribute is None" + assert len(sources) > 0, "sources list is empty" + + policy_ref = f"ratelimitpolicy.kuadrant.io:{rate_limit.namespace()}/{rate_limit.name()}" + assert policy_ref in sources, f"RateLimitPolicy {policy_ref} not found in sources: {sources}" + + assert limitador_span.has_tag("name"), "limitador span missing 'name' attribute" + assert limitador_span.has_tag("namespace"), "limitador span missing 'namespace' attribute" + + name = limitador_span.get_tag("name") + namespace = limitador_span.get_tag("namespace") + assert name is not None and name != "", "limitador name attribute is empty" + assert namespace is not None and namespace != "", "limitador namespace attribute is empty" + + +def test_authconfig_span_is_child_of_reconciler(authconfig_trace): + """ + Validate that authconfig spans are children of reconciler.auth_configs spans. + """ + authconfig_span = authconfig_trace.filter_spans( + lambda s: s.operation_name == "authconfig" and s.has_tag("sources") + )[0] + + parent_id = authconfig_span.get_parent_id() + assert parent_id is not None, "authconfig span has no parent" + + parent_span = authconfig_trace.get_span_by_id(parent_id) + assert parent_span is not None, f"Parent span with ID {parent_id} not found in trace" + assert ( + parent_span.operation_name == "reconciler.auth_configs" + ), f"Expected parent to be 'reconciler.auth_configs' but got '{parent_span.operation_name}'" + + +@pytest.fixture(scope="function") +def second_auth_policy(request, cluster, blame, route, module_label): + """Create a second AuthPolicy targeting the same route""" + # Capture timestamp before creating the second policy + create_time = int(time.time() * 1_000_000) + + second_policy = AuthPolicy.create_instance(cluster, blame("second-auth"), route, labels={"app": module_label}) + second_policy.identity.add_api_key("second_key", Selector(matchLabels={"app": module_label})) + request.addfinalizer(second_policy.delete) + second_policy.commit() + second_policy.wait_for_ready() + return second_policy, create_time + + +@pytest.fixture(scope="function") +def authconfig_trace_multiple_policies(authorization, second_auth_policy, tracing, skip_or_fail): + """Find trace with authconfig span containing sources from multiple policies""" + second_policy, create_time = second_auth_policy + + # Query for traces that started after the second policy was created + # The backoff decorator will retry until traces appear + traces = tracing.get_traces(service="kuadrant-operator", start_time=create_time) + + # Look for a trace that has authconfig span with at least one policy in sources + first_policy_ref = f"authpolicy.kuadrant.io:{authorization.namespace()}/{authorization.name()}" + second_policy_ref = f"authpolicy.kuadrant.io:{second_policy.namespace()}/{second_policy.name()}" + + for trace in traces: + spans = trace.filter_spans(lambda s: s.operation_name == "authconfig" and s.has_tag("sources")) + for span in spans: + sources = span.get_tag("sources") + if sources and (first_policy_ref in sources or second_policy_ref in sources): + return trace + + skip_or_fail( + f"No trace with authconfig span found with either policy. " + f"Looking for {first_policy_ref} or {second_policy_ref} in sources" + ) + + +def test_authconfig_sources_contains_multiple_policies( + authconfig_trace_multiple_policies, authorization, second_auth_policy +): + """ + Validate that when multiple AuthPolicies target the same route, + the authconfig span's sources attribute contains at least one of them. + """ + second_policy, _ = second_auth_policy + + authconfig_spans = authconfig_trace_multiple_policies.filter_spans( + lambda s: s.operation_name == "authconfig" and s.has_tag("sources") + ) + + assert len(authconfig_spans) > 0, "No authconfig spans with sources found" + + first_policy_ref = f"authpolicy.kuadrant.io:{authorization.namespace()}/{authorization.name()}" + second_policy_ref = f"authpolicy.kuadrant.io:{second_policy.namespace()}/{second_policy.name()}" + + # Check all authconfig spans with sources to find one with our policies + found = False + for span in authconfig_spans: + sources = span.get_tag("sources") + assert len(sources) > 0, f"sources list is empty for span {span.span_id}" + + if first_policy_ref in sources or second_policy_ref in sources: + found = True + break + + assert found, ( + f"Neither {first_policy_ref} nor {second_policy_ref} found in any authconfig span sources. " + f"Checked {len(authconfig_spans)} span(s)" + ) + + +@pytest.fixture(scope="function") +def second_rate_limit_policy(request, cluster, blame, route, module_label): + """Create a second RateLimitPolicy targeting the same route""" + # Capture timestamp before creating the second policy + create_time = int(time.time() * 1_000_000) + + second_policy = RateLimitPolicy.create_instance(cluster, blame("second-rlp"), route, labels={"app": module_label}) + second_policy.add_limit("basic", [Limit(10, "10s")]) + request.addfinalizer(second_policy.delete) + second_policy.commit() + second_policy.wait_for_ready() + return second_policy, create_time + + +@pytest.fixture(scope="function") +def limitador_trace_multiple_policies(rate_limit, second_rate_limit_policy, tracing, skip_or_fail): + """Find trace with span containing sources from multiple rate limit policies""" + second_policy, create_time = second_rate_limit_policy + + # Query for traces that started after the second policy was created + # The backoff decorator will retry until traces appear + traces = tracing.get_traces(service="kuadrant-operator", start_time=create_time) + + # Look for a trace that has span with at least one policy in sources + first_policy_ref = f"ratelimitpolicy.kuadrant.io:{rate_limit.namespace()}/{rate_limit.name()}" + second_policy_ref = f"ratelimitpolicy.kuadrant.io:{second_policy.namespace()}/{second_policy.name()}" + + for trace in traces: + spans = trace.filter_spans(lambda s: s.has_tag("sources")) + for span in spans: + sources = span.get_tag("sources") + if sources and (first_policy_ref in sources or second_policy_ref in sources): + return trace + + skip_or_fail(f"No trace found with either policy. Looking for {first_policy_ref} or {second_policy_ref} in sources") + + +def test_limitador_sources_contains_multiple_policies( + limitador_trace_multiple_policies, rate_limit, second_rate_limit_policy +): + """ + Validate that when multiple RateLimitPolicies target the same route, + the limitador limits span's sources attribute contains at least one of them. + """ + second_policy, _ = second_rate_limit_policy + + spans_with_sources = limitador_trace_multiple_policies.filter_spans(lambda s: s.has_tag("sources")) + + assert len(spans_with_sources) > 0, "No spans with sources found" + + first_policy_ref = f"ratelimitpolicy.kuadrant.io:{rate_limit.namespace()}/{rate_limit.name()}" + second_policy_ref = f"ratelimitpolicy.kuadrant.io:{second_policy.namespace()}/{second_policy.name()}" + + # Check all spans with sources to find one with our policies + found = False + for span in spans_with_sources: + sources = span.get_tag("sources") + assert len(sources) > 0, f"sources list is empty for span {span.span_id}" + + if first_policy_ref in sources or second_policy_ref in sources: + found = True + break + + assert found, ( + f"Neither {first_policy_ref} nor {second_policy_ref} found in any span sources. " + f"Checked {len(spans_with_sources)} span(s)" + ) \ No newline at end of file From cdb045162ecbf905a990dffb7fa731de074fd4c0 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Tue, 7 Apr 2026 17:52:48 +0200 Subject: [PATCH 6/9] fix: cosmetic changes. Signed-off-by: Alexander Cristurean --- .../tracing/control_plane/test_control_plane_sources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testsuite/tests/singlecluster/tracing/control_plane/test_control_plane_sources.py b/testsuite/tests/singlecluster/tracing/control_plane/test_control_plane_sources.py index 34b220457..a47ccb855 100644 --- a/testsuite/tests/singlecluster/tracing/control_plane/test_control_plane_sources.py +++ b/testsuite/tests/singlecluster/tracing/control_plane/test_control_plane_sources.py @@ -246,4 +246,4 @@ def test_limitador_sources_contains_multiple_policies( assert found, ( f"Neither {first_policy_ref} nor {second_policy_ref} found in any span sources. " f"Checked {len(spans_with_sources)} span(s)" - ) \ No newline at end of file + ) From e25f0336a5e3cca9caf8a1ad06634a04c703aed0 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Wed, 8 Apr 2026 08:48:11 +0200 Subject: [PATCH 7/9] fix: cosmetic changes. Signed-off-by: Alexander Cristurean --- .../control_plane/test_control_plane_sources.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/testsuite/tests/singlecluster/tracing/control_plane/test_control_plane_sources.py b/testsuite/tests/singlecluster/tracing/control_plane/test_control_plane_sources.py index a47ccb855..90cfb7662 100644 --- a/testsuite/tests/singlecluster/tracing/control_plane/test_control_plane_sources.py +++ b/testsuite/tests/singlecluster/tracing/control_plane/test_control_plane_sources.py @@ -24,7 +24,7 @@ def authconfig_trace(auth_traces, skip_or_fail): if spans: return trace - skip_or_fail("No trace with authconfig span containing 'sources' attribute found") + return skip_or_fail("No trace with authconfig span containing 'sources' attribute found") @pytest.fixture(scope="module") @@ -36,7 +36,7 @@ def limitador_trace(rl_traces, skip_or_fail): return trace available_operations = list({span.operation_name for trace in rl_traces for span in trace.spans})[:20] - skip_or_fail( + return skip_or_fail( f"No trace with spans containing 'sources' attribute found in rate limit traces. " f"Available operations: {available_operations}" ) @@ -140,7 +140,7 @@ def authconfig_trace_multiple_policies(authorization, second_auth_policy, tracin if sources and (first_policy_ref in sources or second_policy_ref in sources): return trace - skip_or_fail( + return skip_or_fail( f"No trace with authconfig span found with either policy. " f"Looking for {first_policy_ref} or {second_policy_ref} in sources" ) @@ -214,7 +214,9 @@ def limitador_trace_multiple_policies(rate_limit, second_rate_limit_policy, trac if sources and (first_policy_ref in sources or second_policy_ref in sources): return trace - skip_or_fail(f"No trace found with either policy. Looking for {first_policy_ref} or {second_policy_ref} in sources") + return skip_or_fail( + f"No trace found with either policy. Looking for {first_policy_ref} or {second_policy_ref} in sources" + ) def test_limitador_sources_contains_multiple_policies( From 40ad6bc16abfac710793c63b727ca9275f265e0c Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Wed, 8 Apr 2026 09:19:46 +0200 Subject: [PATCH 8/9] fix: remove unused lookback and improve skip message. Signed-off-by: Alexander Cristurean --- .../tracing/control_plane/test_control_plane_sources.py | 6 +----- testsuite/tracing/__init__.py | 2 -- testsuite/tracing/jaeger.py | 6 +----- testsuite/tracing/tempo.py | 5 +---- 4 files changed, 3 insertions(+), 16 deletions(-) diff --git a/testsuite/tests/singlecluster/tracing/control_plane/test_control_plane_sources.py b/testsuite/tests/singlecluster/tracing/control_plane/test_control_plane_sources.py index 90cfb7662..ca9af0e27 100644 --- a/testsuite/tests/singlecluster/tracing/control_plane/test_control_plane_sources.py +++ b/testsuite/tests/singlecluster/tracing/control_plane/test_control_plane_sources.py @@ -35,11 +35,7 @@ def limitador_trace(rl_traces, skip_or_fail): if spans: return trace - available_operations = list({span.operation_name for trace in rl_traces for span in trace.spans})[:20] - return skip_or_fail( - f"No trace with spans containing 'sources' attribute found in rate limit traces. " - f"Available operations: {available_operations}" - ) + return skip_or_fail("No trace with limits span containing 'sources' attribute found") def test_authconfig_span_attributes(authconfig_trace, authorization): diff --git a/testsuite/tracing/__init__.py b/testsuite/tracing/__init__.py index 5bfe20df0..6905431cf 100644 --- a/testsuite/tracing/__init__.py +++ b/testsuite/tracing/__init__.py @@ -28,10 +28,8 @@ def get_traces( service: str, tags: Optional[dict[str, str]] = None, min_processes: int = 0, - lookback: Optional[str] = None, start_time: Optional[int] = None, ) -> list[Any]: """Search traces in tracing client by service name and tags. If min_processes is set, retries until at least that many service processes are present. - If lookback is set, only returns traces within that time window (e.g., "1h", "30m"). If start_time is set, only returns traces that started after that time (in microseconds).""" diff --git a/testsuite/tracing/jaeger.py b/testsuite/tracing/jaeger.py index 278455e3a..6d66ed296 100644 --- a/testsuite/tracing/jaeger.py +++ b/testsuite/tracing/jaeger.py @@ -38,7 +38,6 @@ def get_traces( service: str, tags: Optional[dict[str, str]] = None, min_processes: int = 0, - lookback: Optional[str] = None, start_time: Optional[int] = None, ) -> list[Trace]: """Gets trace from tracing backend Tempo or Jaeger. @@ -48,7 +47,6 @@ def get_traces( service: Service name to filter traces tags: Optional tags to filter traces min_processes: Minimum number of processes required in traces - lookback: Optional lookback duration (e.g., "1h", "30m", "1m") start_time: Optional start time in microseconds (filters traces that started after this time) Returns: @@ -57,10 +55,8 @@ def get_traces( params = {"service": service} if tags: params["tags"] = json.dumps(tags) - if lookback: - params["lookback"] = lookback if start_time: - params["start"] = start_time + params["start"] = str(start_time) traces_data = self.query.api.traces.get(params=params).json()["data"] if not traces_data: diff --git a/testsuite/tracing/tempo.py b/testsuite/tracing/tempo.py index 97456633b..9493a2ce9 100644 --- a/testsuite/tracing/tempo.py +++ b/testsuite/tracing/tempo.py @@ -17,7 +17,6 @@ def get_traces( service: str, tags: Optional[dict[str, str]] = None, min_processes: int = 0, - lookback: Optional[str] = None, start_time: Optional[int] = None, ) -> list[Trace]: """Gets trace from Tempo tracing backend. @@ -25,10 +24,8 @@ def get_traces( params = {"service.name": service} if tags: params.update(tags) - if lookback: - params["lookback"] = lookback if start_time: - params["start"] = start_time + params["start"] = str(start_time) traces_data = self.query.api.get_traces.get(params=params).json()["traces"] if not traces_data: return [] From bb940428c161e49fdee0c8bf02e8a6ce98e9c49d Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Tue, 21 Apr 2026 12:00:28 +0200 Subject: [PATCH 9/9] fix: fix span filter. Signed-off-by: Alexander Cristurean --- .../tracing/control_plane/test_control_plane_sources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testsuite/tests/singlecluster/tracing/control_plane/test_control_plane_sources.py b/testsuite/tests/singlecluster/tracing/control_plane/test_control_plane_sources.py index ca9af0e27..1df6fd7e3 100644 --- a/testsuite/tests/singlecluster/tracing/control_plane/test_control_plane_sources.py +++ b/testsuite/tests/singlecluster/tracing/control_plane/test_control_plane_sources.py @@ -31,7 +31,7 @@ def authconfig_trace(auth_traces, skip_or_fail): def limitador_trace(rl_traces, skip_or_fail): """Find trace with span that has sources attribute""" for trace in rl_traces: - spans = trace.filter_spans(lambda s: s.operation_name == "limits" and s.has_tag("sources")) + spans = trace.filter_spans(lambda s: s.operation_name == "reconciler.limitador_limits" and s.has_tag("sources")) if spans: return trace