From 2e24c437960ddaf3b36251e4a362eec7ccda88ac Mon Sep 17 00:00:00 2001 From: Ali Aqel Date: Sun, 12 Apr 2026 23:42:39 +0000 Subject: [PATCH 01/22] feat: add HTTP client request metrics for all KEDA scaler outbound requests Instruments all outbound HTTP calls made by KEDA scalers with keda_http_client_requests_total and keda_http_client_request_duration_seconds (Prometheus) and keda.http.client.requests.count / keda.http.client.request.duration.seconds (OTel) via an InstrumentedRoundTripper injected in CreateHTTPTransportWithTLSConfig. Closes #6600 Signed-off-by: Ali Aqel --- CHANGELOG.md | 1 + pkg/metricscollector/metricscollectors.go | 14 ++ pkg/metricscollector/opentelemetry.go | 35 +++++ pkg/metricscollector/opentelemetry_test.go | 53 ++++++++ pkg/metricscollector/prommetrics.go | 38 ++++++ pkg/metricscollector/prommetrics_test.go | 74 +++++++++++ pkg/scaling/cache/scalers_cache.go | 6 + pkg/util/http.go | 8 +- pkg/util/http_roundtripper.go | 92 +++++++++++++ pkg/util/http_roundtripper_test.go | 123 ++++++++++++++++++ .../opentelemetry_metrics_test.go | 56 ++++++++ .../prometheus_metrics_test.go | 52 ++++++++ 12 files changed, 548 insertions(+), 4 deletions(-) create mode 100644 pkg/metricscollector/prommetrics_test.go create mode 100644 pkg/util/http_roundtripper.go create mode 100644 pkg/util/http_roundtripper_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 49206cb4a7d..ae2f1d2c7a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,6 +76,7 @@ To learn more about active deprecations, we recommend checking [GitHub Discussio ### Improvements +- **General**: Add HTTP client request metrics for all outbound HTTP requests made by KEDA scalers ([#6600](https://github.com/kedacore/keda/issues/6600)) - **General**: Add CRD-level validation markers (Minimum, MinLength, MinItems, Enum) for ScaledObject, ScaledJob, ScaleTriggers, and TriggerAuthentication API types ([#7533](https://github.com/kedacore/keda/pull/7533)) - **General**: Add `--leader-election-id` flag to allow configuring the leader election Lease name ([#7564](https://github.com/kedacore/keda/issues/7564)) - **General**: Make APIService cert injections optional ([#7559](https://github.com/kedacore/keda/pull/7559)) diff --git a/pkg/metricscollector/metricscollectors.go b/pkg/metricscollector/metricscollectors.go index 7a6d1055ffa..ac6bf7efccf 100644 --- a/pkg/metricscollector/metricscollectors.go +++ b/pkg/metricscollector/metricscollectors.go @@ -79,6 +79,11 @@ type MetricsCollector interface { // RecordCloudEventQueueStatus record the number of cloudevents that are waiting for emitting RecordCloudEventQueueStatus(namespace string, value int) + + // RecordHTTPClientRequest records the duration and outcome of an outbound HTTP request + // made by one of KEDA's internal HTTP clients. scaler, triggerName, and metricName + // are read from context keys set in util package; empty string if unset. + RecordHTTPClientRequest(durationSeconds float64, statusCode int, isError bool, scaler, triggerName, metricName, namespace, scaledResource string) } func NewMetricsCollectors(enablePrometheusMetrics bool, enableOpenTelemetryMetrics bool) { @@ -205,6 +210,15 @@ func RecordCloudEventQueueStatus(namespace string, value int) { } } +// RecordHTTPClientRequest records the duration and outcome of an outbound HTTP request +// made by one of KEDA's internal HTTP clients. scaler, triggerName, and metricName +// are read from context keys set in util package; empty string if unset. +func RecordHTTPClientRequest(durationSeconds float64, statusCode int, isError bool, scaler, triggerName, metricName, namespace, scaledResource string) { + for _, element := range collectors { + element.RecordHTTPClientRequest(durationSeconds, statusCode, isError, scaler, triggerName, metricName, namespace, scaledResource) + } +} + // Returns the ServerMetrics object for GRPC Server metrics. Used to initialize the GRPC server with the proper intercepts // Currently, only Prometheus metrics are supported. func GetServerMetrics() *grpcprom.ServerMetrics { diff --git a/pkg/metricscollector/opentelemetry.go b/pkg/metricscollector/opentelemetry.go index 14e2aa32f47..592a2830b2d 100644 --- a/pkg/metricscollector/opentelemetry.go +++ b/pkg/metricscollector/opentelemetry.go @@ -47,6 +47,9 @@ var ( otelScalerActiveVals []OtelMetricFloat64Val otelScalerPauseVals []OtelMetricFloat64Val + + otHTTPClientRequestsCounter api.Int64Counter + otHTTPClientRequestDuration api.Float64Histogram ) type OtelMetrics struct { @@ -220,6 +223,23 @@ func initMeters() { if err != nil { otLog.Error(err, msg) } + + otHTTPClientRequestsCounter, err = meter.Int64Counter( + "keda.http.client.requests.count", + api.WithDescription("Total number of outbound HTTP requests issued by KEDA's HTTP clients, labeled by status class."), + ) + if err != nil { + otLog.Error(err, msg) + } + + otHTTPClientRequestDuration, err = meter.Float64Histogram( + "keda.http.client.request.duration.seconds", + api.WithDescription("Duration in seconds of outbound HTTP requests issued by KEDA's HTTP clients."), + api.WithUnit("s"), + ) + if err != nil { + otLog.Error(err, msg) + } } func BuildInfoCallback(_ context.Context, obsrv api.Int64Observer) error { @@ -495,6 +515,21 @@ func CloudeventQueueStatusCallback(_ context.Context, obsrv api.Float64Observer) return nil } +// RecordHTTPClientRequest records the duration and outcome of a single outbound HTTP request. +func (o *OtelMetrics) RecordHTTPClientRequest(durationSeconds float64, statusCode int, isError bool, scaler, triggerName, metricName, namespace, scaledResource string) { + code := httpStatusCodeLabel(statusCode, isError) + opt := api.WithAttributes( + attribute.Key("namespace").String(namespace), + attribute.Key("scaled_resource").String(scaledResource), + attribute.Key("scaler").String(scaler), + attribute.Key("trigger_name").String(triggerName), + attribute.Key("metric_name").String(metricName), + attribute.Key("status_code").String(code), + ) + otHTTPClientRequestsCounter.Add(context.Background(), 1, opt) + otHTTPClientRequestDuration.Record(context.Background(), durationSeconds, opt) +} + // RecordCloudEventQueueStatus record the number of cloudevents that are waiting for emitting func (o *OtelMetrics) RecordCloudEventQueueStatus(namespace string, value int) { opt := api.WithAttributes( diff --git a/pkg/metricscollector/opentelemetry_test.go b/pkg/metricscollector/opentelemetry_test.go index 3cd2616f001..4e7d98c2bbe 100644 --- a/pkg/metricscollector/opentelemetry_test.go +++ b/pkg/metricscollector/opentelemetry_test.go @@ -103,6 +103,59 @@ func TestLoopLatency(t *testing.T) { assert.Equal(t, data.Value, float64(0.5)) } +func TestRecordHTTPClientRequest(t *testing.T) { + tests := []struct { + name string + statusCode int + isError bool + wantStatusCode string + }{ + {"200 success", 200, false, "200"}, + {"301 redirect", 301, false, "301"}, + {"404 client error", 404, false, "404"}, + {"503 server error", 503, false, "503"}, + {"transport error", 0, true, "error"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + testOtel.RecordHTTPClientRequest(0.1, tt.statusCode, tt.isError, "prometheus", "my-trigger", "my-metric", "default", "my-so") + got := metricdata.ResourceMetrics{} + err := testReader.Collect(context.Background(), &got) + assert.Nil(t, err) + + scopeMetrics := got.ScopeMetrics[0] + requestCount := retrieveMetric(scopeMetrics.Metrics, "keda.http.client.requests.count") + assert.NotNil(t, requestCount) + + var found bool + for _, dp := range requestCount.Data.(metricdata.Sum[int64]).DataPoints { + code, ok := dp.Attributes.Value("status_code") + if !ok || code.AsString() != tt.wantStatusCode { + continue + } + scaler, _ := dp.Attributes.Value("scaler") + assert.Equal(t, "prometheus", scaler.AsString()) + triggerName, _ := dp.Attributes.Value("trigger_name") + assert.Equal(t, "my-trigger", triggerName.AsString()) + metricName, _ := dp.Attributes.Value("metric_name") + assert.Equal(t, "my-metric", metricName.AsString()) + ns, _ := dp.Attributes.Value("namespace") + assert.Equal(t, "default", ns.AsString()) + sr, _ := dp.Attributes.Value("scaled_resource") + assert.Equal(t, "my-so", sr.AsString()) + found = true + break + } + assert.True(t, found, "expected data point with status_code=%q", tt.wantStatusCode) + + requestDuration := retrieveMetric(scopeMetrics.Metrics, "keda.http.client.request.duration.seconds") + assert.NotNil(t, requestDuration) + assert.Equal(t, "s", requestDuration.Unit) + }) + } +} + func TestContinuousMetrics(t *testing.T) { testOtel.RecordScalerActive("testnamespace", "testresource", "testscaler", 0, "testmetric", true, true) testOtel.RecordScalerActive("testnamespace2", "testresource2", "testscaler2", 0, "testmetric", false, false) diff --git a/pkg/metricscollector/prommetrics.go b/pkg/metricscollector/prommetrics.go index 92b60c17f26..1dc9d0e3dfc 100644 --- a/pkg/metricscollector/prommetrics.go +++ b/pkg/metricscollector/prommetrics.go @@ -152,6 +152,27 @@ var ( }, []string{"namespace"}, ) + + httpClientRequestsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: DefaultPromMetricsNamespace, + Subsystem: "http_client", + Name: "requests_total", + Help: "Total number of outbound HTTP requests issued by KEDA's HTTP clients.", + }, + []string{"namespace", "scaled_resource", "scaler", "trigger_name", "metric_name", "status_code"}, + ) + + httpClientRequestDuration = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: DefaultPromMetricsNamespace, + Subsystem: "http_client", + Name: "request_duration_seconds", + Help: "Duration in seconds of outbound HTTP requests issued by KEDA's HTTP clients.", + Buckets: []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10}, + }, + []string{"namespace", "scaled_resource", "scaler", "trigger_name", "metric_name", "status_code"}, + ) ) type PromMetrics struct { @@ -174,6 +195,9 @@ func NewPromMetrics() *PromMetrics { metrics.Registry.MustRegister(cloudeventEmitted) metrics.Registry.MustRegister(cloudeventQueueStatus) + metrics.Registry.MustRegister(httpClientRequestsTotal) + metrics.Registry.MustRegister(httpClientRequestDuration) + RecordBuildInfo() return &PromMetrics{} } @@ -328,6 +352,20 @@ func (p *PromMetrics) RecordCloudEventQueueStatus(namespace string, value int) { cloudeventQueueStatus.With(prometheus.Labels{"namespace": namespace}).Set(float64(value)) } +// RecordHTTPClientRequest records the duration and outcome of a single outbound HTTP request. +func (p *PromMetrics) RecordHTTPClientRequest(durationSeconds float64, statusCode int, isError bool, scaler, triggerName, metricName, namespace, scaledResource string) { + code := httpStatusCodeLabel(statusCode, isError) + httpClientRequestsTotal.WithLabelValues(namespace, scaledResource, scaler, triggerName, metricName, code).Inc() + httpClientRequestDuration.WithLabelValues(namespace, scaledResource, scaler, triggerName, metricName, code).Observe(durationSeconds) +} + +func httpStatusCodeLabel(code int, isError bool) string { + if isError { + return "error" + } + return strconv.Itoa(code) +} + // Returns a grpcprom server Metrics object and registers the metrics. The object contains // interceptors to chain to the server so that all requests served are observed. Intended to be called // as part of initialization of metricscollector, hence why this function is not exported diff --git a/pkg/metricscollector/prommetrics_test.go b/pkg/metricscollector/prommetrics_test.go new file mode 100644 index 00000000000..478207ba011 --- /dev/null +++ b/pkg/metricscollector/prommetrics_test.go @@ -0,0 +1,74 @@ +/* +Copyright 2025 The KEDA Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metricscollector + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestHTTPStatusCodeLabel(t *testing.T) { + tests := []struct { + name string + statusCode int + isError bool + want string + }{ + {"transport error", 0, true, "error"}, + {"200 OK", 200, false, "200"}, + {"201 Created", 201, false, "201"}, + {"301 Moved", 301, false, "301"}, + {"400 Bad Request", 400, false, "400"}, + {"404 Not Found", 404, false, "404"}, + {"500 Internal Server Error", 500, false, "500"}, + {"503 Service Unavailable", 503, false, "503"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := httpStatusCodeLabel(tt.statusCode, tt.isError) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestPromMetrics_RecordHTTPClientRequest(t *testing.T) { + p := &PromMetrics{} + + // Verify no panic and label combinations are created without error. + p.RecordHTTPClientRequest(0.05, 200, false, "prometheus", "my-trigger", "my-metric", "default", "my-so") + p.RecordHTTPClientRequest(0.1, 404, false, "redis", "redis-trigger", "redis-metric", "default", "my-so") + p.RecordHTTPClientRequest(0.2, 500, false, "prometheus", "my-trigger", "my-metric", "default", "my-so") + p.RecordHTTPClientRequest(0.3, 0, true, "", "", "", "", "") + + counter, err := httpClientRequestsTotal.GetMetricWithLabelValues("default", "my-so", "prometheus", "my-trigger", "my-metric", "200") + assert.NoError(t, err) + assert.NotNil(t, counter) + + counter, err = httpClientRequestsTotal.GetMetricWithLabelValues("default", "my-so", "redis", "redis-trigger", "redis-metric", "404") + assert.NoError(t, err) + assert.NotNil(t, counter) + + counter, err = httpClientRequestsTotal.GetMetricWithLabelValues("default", "my-so", "prometheus", "my-trigger", "my-metric", "500") + assert.NoError(t, err) + assert.NotNil(t, counter) + + counter, err = httpClientRequestsTotal.GetMetricWithLabelValues("", "", "", "", "", "error") + assert.NoError(t, err) + assert.NotNil(t, counter) +} diff --git a/pkg/scaling/cache/scalers_cache.go b/pkg/scaling/cache/scalers_cache.go index d7297b25d48..1054d29819c 100644 --- a/pkg/scaling/cache/scalers_cache.go +++ b/pkg/scaling/cache/scalers_cache.go @@ -31,6 +31,7 @@ import ( kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" "github.com/kedacore/keda/v2/pkg/scalers" "github.com/kedacore/keda/v2/pkg/scalers/scalersconfig" + kedautil "github.com/kedacore/keda/v2/pkg/util" ) var log = logf.Log.WithName("scalers_cache") @@ -156,6 +157,11 @@ func (c *ScalersCache) GetMetricsAndActivityForScaler(ctx context.Context, index if err != nil { return nil, false, -1, err } + ctx = context.WithValue(ctx, kedautil.ScalerContextKey, sb.ScalerConfig.TriggerType) + ctx = context.WithValue(ctx, kedautil.TriggerNameContextKey, sb.ScalerConfig.TriggerName) + ctx = context.WithValue(ctx, kedautil.MetricNameContextKey, metricName) + ctx = context.WithValue(ctx, kedautil.NamespaceContextKey, sb.ScalerConfig.ScalableObjectNamespace) + ctx = context.WithValue(ctx, kedautil.ScaledResourceContextKey, sb.ScalerConfig.ScalableObjectName) startTime := time.Now() metric, activity, err := sb.Scaler.GetMetricsAndActivity(ctx, metricName) if err == nil { diff --git a/pkg/util/http.go b/pkg/util/http.go index 422d91f2098..adc35c6728f 100644 --- a/pkg/util/http.go +++ b/pkg/util/http.go @@ -60,13 +60,13 @@ func CreateHTTPClient(timeout time.Duration, unsafeSsl bool) *http.Client { // CreateHTTPTransport returns a new HTTP Transport with Proxy, Keep alives // unsafeSsl parameter allows to avoid tls cert validation if it's required -func CreateHTTPTransport(unsafeSsl bool) *http.Transport { +func CreateHTTPTransport(unsafeSsl bool) http.RoundTripper { return CreateHTTPTransportWithTLSConfig(CreateTLSClientConfig(unsafeSsl)) } // CreateHTTPTransportWithTLSConfig returns a new HTTP Transport with Proxy, Keep alives -// using given tls.Config -func CreateHTTPTransportWithTLSConfig(config *tls.Config) *http.Transport { +// using given tls.Config, wrapped with instrumentation for HTTP client metrics. +func CreateHTTPTransportWithTLSConfig(config *tls.Config) http.RoundTripper { transport := &http.Transport{ TLSClientConfig: config, Proxy: http.ProxyFromEnvironment, @@ -76,5 +76,5 @@ func CreateHTTPTransportWithTLSConfig(config *tls.Config) *http.Transport { transport.DisableKeepAlives = true transport.IdleConnTimeout = 100 * time.Second } - return transport + return NewInstrumentedRoundTripper(transport) } diff --git a/pkg/util/http_roundtripper.go b/pkg/util/http_roundtripper.go new file mode 100644 index 00000000000..6451e7d85d7 --- /dev/null +++ b/pkg/util/http_roundtripper.go @@ -0,0 +1,92 @@ +/* +Copyright 2025 The KEDA Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "net/http" + "time" + + "github.com/kedacore/keda/v2/pkg/metricscollector" +) + +// contextKey is an unexported type for context keys defined in this package, +// preventing collisions with keys from other packages. +type contextKey string + +const ( + // ScalerContextKey is the context key used to attach the scaler type name + // (e.g. "prometheus", "redis") to an outbound HTTP request so that metrics + // observers can include it as a dimension. + ScalerContextKey contextKey = "scaler" + + // TriggerNameContextKey is the context key used to attach the user-defined + // trigger name to an outbound HTTP request. + TriggerNameContextKey contextKey = "trigger_name" + + // MetricNameContextKey is the context key used to attach the metric name + // being queried to an outbound HTTP request. + MetricNameContextKey contextKey = "metric_name" + + // NamespaceContextKey is the context key used to attach the namespace of the + // ScaledObject/ScaledJob that owns the scaler making the request. + NamespaceContextKey contextKey = "namespace" + + // ScaledResourceContextKey is the context key used to attach the name of the + // ScaledObject/ScaledJob that owns the scaler making the request. + ScaledResourceContextKey contextKey = "scaled_resource" +) + +// InstrumentedRoundTripper wraps an http.RoundTripper and records outbound +// HTTP request metrics after every completed round-trip. It reads known +// context keys from the request context to populate metric dimensions. It +// does not buffer or inspect the response body. +type InstrumentedRoundTripper struct { + next http.RoundTripper +} + +// NewInstrumentedRoundTripper wraps next with a RoundTripper that records +// HTTP request metrics after every request. If next is nil, +// http.DefaultTransport is used. +func NewInstrumentedRoundTripper(next http.RoundTripper) http.RoundTripper { + if next == nil { + next = http.DefaultTransport + } + return &InstrumentedRoundTripper{next: next} +} + +func (r *InstrumentedRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + start := time.Now() + resp, err := r.next.RoundTrip(req) + duration := time.Since(start).Seconds() + + ctx := req.Context() + scaler, ok := ctx.Value(ScalerContextKey).(string) + if !ok || scaler == "" { + scaler = "unknown" + } + triggerName, _ := ctx.Value(TriggerNameContextKey).(string) + metricName, _ := ctx.Value(MetricNameContextKey).(string) + namespace, _ := ctx.Value(NamespaceContextKey).(string) + scaledResource, _ := ctx.Value(ScaledResourceContextKey).(string) + + if err != nil { + metricscollector.RecordHTTPClientRequest(duration, 0, true, scaler, triggerName, metricName, namespace, scaledResource) + return nil, err + } + metricscollector.RecordHTTPClientRequest(duration, resp.StatusCode, false, scaler, triggerName, metricName, namespace, scaledResource) + return resp, nil +} diff --git a/pkg/util/http_roundtripper_test.go b/pkg/util/http_roundtripper_test.go new file mode 100644 index 00000000000..d7e3ea971d5 --- /dev/null +++ b/pkg/util/http_roundtripper_test.go @@ -0,0 +1,123 @@ +/* +Copyright 2025 The KEDA Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "context" + "errors" + "io" + "net/http" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type mockRoundTripper struct { + resp *http.Response + err error +} + +func (m *mockRoundTripper) RoundTrip(_ *http.Request) (*http.Response, error) { + return m.resp, m.err +} + +func fakeResponse(statusCode int) *http.Response { + return &http.Response{ + StatusCode: statusCode, + Body: io.NopCloser(strings.NewReader("")), + } +} + +func newRequest(ctx context.Context) *http.Request { + req, _ := http.NewRequestWithContext(ctx, http.MethodGet, "http://example.com", nil) + return req +} + +func TestInstrumentedRoundTripper_SuccessfulRequest(t *testing.T) { + for _, statusCode := range []int{200, 201, 301, 400, 500} { + t.Run(http.StatusText(statusCode), func(t *testing.T) { + rt := NewInstrumentedRoundTripper(&mockRoundTripper{resp: fakeResponse(statusCode)}) + + resp, err := rt.RoundTrip(newRequest(context.Background())) + + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, statusCode, resp.StatusCode) + }) + } +} + +func TestInstrumentedRoundTripper_TransportError(t *testing.T) { + transportErr := errors.New("connection refused") + rt := NewInstrumentedRoundTripper(&mockRoundTripper{err: transportErr}) + + resp, err := rt.RoundTrip(newRequest(context.Background())) //nolint:bodyclose // resp is nil on error + + assert.ErrorIs(t, err, transportErr) + assert.Nil(t, resp) +} + +func TestInstrumentedRoundTripper_ResponseReturnedUnmodified(t *testing.T) { + expected := fakeResponse(202) + rt := NewInstrumentedRoundTripper(&mockRoundTripper{resp: expected}) + + got, err := rt.RoundTrip(newRequest(context.Background())) + + require.NoError(t, err) + defer got.Body.Close() + assert.Same(t, expected, got) +} + +func TestInstrumentedRoundTripper_NilNextUsesDefault(t *testing.T) { + rt := NewInstrumentedRoundTripper(nil) + irt, ok := rt.(*InstrumentedRoundTripper) + require.True(t, ok) + assert.Equal(t, http.DefaultTransport, irt.next) +} + +func TestInstrumentedRoundTripper_ScalerContextKey_Missing(t *testing.T) { + // When ScalerContextKey is absent the RoundTripper should not panic and + // should complete normally — "unknown" is used as the scaler label value. + rt := NewInstrumentedRoundTripper(&mockRoundTripper{resp: fakeResponse(200)}) + + resp, err := rt.RoundTrip(newRequest(context.Background())) + + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, 200, resp.StatusCode) +} + +func TestInstrumentedRoundTripper_ScalerContextKey(t *testing.T) { + // Requests with ScalerContextKey set should not panic or error — the key + // is read inside RoundTrip and forwarded to the metrics collector. + rt := NewInstrumentedRoundTripper(&mockRoundTripper{resp: fakeResponse(200)}) + + ctx := context.WithValue(context.Background(), ScalerContextKey, "prometheus") + resp, err := rt.RoundTrip(newRequest(ctx)) + + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, 200, resp.StatusCode) +} + +func TestCreateHTTPClient_TransportIsInstrumented(t *testing.T) { + client := CreateHTTPClient(0, false) + _, ok := client.Transport.(*InstrumentedRoundTripper) + assert.True(t, ok, "expected CreateHTTPClient to wrap transport with InstrumentedRoundTripper") +} diff --git a/tests/sequential/opentelemetry_metrics/opentelemetry_metrics_test.go b/tests/sequential/opentelemetry_metrics/opentelemetry_metrics_test.go index b000f63fb06..cacd712ff8a 100644 --- a/tests/sequential/opentelemetry_metrics/opentelemetry_metrics_test.go +++ b/tests/sequential/opentelemetry_metrics/opentelemetry_metrics_test.go @@ -487,6 +487,7 @@ func TestOpenTelemetryMetrics(t *testing.T) { testScaledObjectPausedMetric(t, data) testCloudEventEmitted(t, data) testCloudEventEmittedError(t, data) + testHTTPClientMetrics(t, data) changeOtlpProtocolInOperator(t, kc, "keda-operator", "keda") testScalerGrpcMetricValue(t, kc, data) @@ -1263,3 +1264,58 @@ func testCloudEventEmittedError(t *testing.T, data templateData) { KubectlDeleteWithTemplate(t, data, "wrongCloudEventSourceTemplate", wrongCloudEventSourceTemplate) KubectlApplyWithTemplate(t, data, "cloudEventSourceTemplate", cloudEventSourceTemplate) } + +func testHTTPClientMetrics(t *testing.T, data templateData) { + t.Log("--- testing HTTP client metrics ---") + + // The wrongScaledObject uses a prometheus-type scaler that makes real HTTP + // requests on every poll interval, so its records should be present once at + // least one poll cycle has completed. + KubectlDeleteWithTemplate(t, data, "scaledObjectTemplate", scaledObjectTemplate) + KubectlApplyWithTemplate(t, data, "wrongScaledObjectTemplate", wrongScaledObjectTemplate) + defer func() { + KubectlDeleteWithTemplate(t, data, "wrongScaledObjectTemplate", wrongScaledObjectTemplate) + KubectlApplyWithTemplate(t, data, "scaledObjectTemplate", scaledObjectTemplate) + }() + + time.Sleep(20 * time.Second) + + family := fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure %s", kedaOperatorCollectorPrometheusExportURL)) + + matchLabels := func(labels []*prommodel.LabelPair) bool { + return ExtractPrometheusLabelValue("namespace", labels) == data.TestNamespace && + ExtractPrometheusLabelValue("scaled_resource", labels) == wrongScaledObjectName && + ExtractPrometheusLabelValue("scaler", labels) == "prometheus" && + ExtractPrometheusLabelValue("trigger_name", labels) == wrongScalerName && + ExtractPrometheusLabelValue("metric_name", labels) == "s0-prometheus" + } + + val, ok := family["keda_http_client_requests_count_total"] + assert.True(t, ok, "keda_http_client_requests_count_total not available") + if ok { + var found bool + for _, metric := range val.GetMetric() { + if matchLabels(metric.GetLabel()) { + assert.GreaterOrEqual(t, metric.GetCounter().GetValue(), float64(1)) + found = true + break + } + } + assert.True(t, found, + "expected keda_http_client_requests_count_total with namespace=%s, scaled_resource=%s, scaler=prometheus, trigger_name=%s, metric_name=s0-prometheus", + data.TestNamespace, wrongScaledObjectName, wrongScalerName) + } + + if val, ok := family["keda_http_client_request_duration_seconds"]; ok { + var found bool + for _, metric := range val.GetMetric() { + if matchLabels(metric.GetLabel()) { + assert.Greater(t, metric.GetHistogram().GetSampleCount(), uint64(0), + "keda_http_client_request_duration_seconds sample count should be > 0") + found = true + break + } + } + assert.True(t, found, "expected keda_http_client_request_duration_seconds histogram for prometheus scaler") + } +} diff --git a/tests/sequential/prometheus_metrics/prometheus_metrics_test.go b/tests/sequential/prometheus_metrics/prometheus_metrics_test.go index 58152d979d0..e68c8862fc5 100644 --- a/tests/sequential/prometheus_metrics/prometheus_metrics_test.go +++ b/tests/sequential/prometheus_metrics/prometheus_metrics_test.go @@ -450,6 +450,7 @@ func TestPrometheusMetrics(t *testing.T) { testScaledObjectPausedMetric(t, data) testCloudEventEmitted(t, data) testCloudEventEmittedError(t, data) + testHTTPClientMetrics(t, data) // cleanup DeleteKubernetesResources(t, testNamespace, data, templates) } @@ -1430,3 +1431,54 @@ func testCloudEventEmittedError(t *testing.T, data templateData) { assert.True(t, familyValidator(metric)) } + +func testHTTPClientMetrics(t *testing.T, data templateData) { + t.Log("--- testing HTTP client metrics ---") + + // The wrongScaledObject uses a prometheus-type scaler that makes real HTTP + // requests on every poll interval, so its records should be present once at + // least one poll cycle has completed. + KubectlDeleteWithTemplate(t, data, "scaledObjectTemplate", scaledObjectTemplate) + KubectlApplyWithTemplate(t, data, "wrongScaledObjectTemplate", wrongScaledObjectTemplate) + defer func() { + KubectlDeleteWithTemplate(t, data, "wrongScaledObjectTemplate", wrongScaledObjectTemplate) + KubectlApplyWithTemplate(t, data, "scaledObjectTemplate", scaledObjectTemplate) + }() + + matchLabels := func(labels []*prommodel.LabelPair) bool { + return ExtractPrometheusLabelValue("namespace", labels) == testNamespace && + ExtractPrometheusLabelValue("scaled_resource", labels) == wrongScaledObjectName && + ExtractPrometheusLabelValue("scaler", labels) == "prometheus" && + ExtractPrometheusLabelValue("trigger_name", labels) == wrongScalerName && + ExtractPrometheusLabelValue("metric_name", labels) == "s0-prometheus" + } + + familyValidator := func(family *prommodel.MetricFamily) bool { + for _, metric := range family.GetMetric() { + if matchLabels(metric.GetLabel()) && metric.GetCounter().GetValue() >= 1 { + return true + } + } + return false + } + + families := WaitForPrometheusMetric(t, "keda_http_client_requests_total", familyValidator) + assert.True(t, familyValidator(families["keda_http_client_requests_total"]), + "expected keda_http_client_requests_total with namespace=%s, scaled_resource=%s, scaler=prometheus, trigger_name=%s, metric_name=s0-prometheus", + testNamespace, wrongScaledObjectName, wrongScalerName) + + family, ok := families["keda_http_client_request_duration_seconds"] + assert.True(t, ok, "keda_http_client_request_duration_seconds not present") + if ok { + var found bool + for _, metric := range family.GetMetric() { + if matchLabels(metric.GetLabel()) { + assert.Greater(t, metric.GetHistogram().GetSampleCount(), uint64(0), + "keda_http_client_request_duration_seconds sample count should be > 0") + found = true + break + } + } + assert.True(t, found, "expected keda_http_client_request_duration_seconds histogram for prometheus scaler") + } +} From b6b64e04bac36f895291f1881fcd5cc9463ce08a Mon Sep 17 00:00:00 2001 From: Ali Aqel Date: Sun, 12 Apr 2026 23:49:23 +0000 Subject: [PATCH 02/22] fix(aws): initialize SigV4 transport once to avoid per-request allocation and double instrumentation Signed-off-by: Ali Aqel --- pkg/scalers/aws/aws_sigv4.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pkg/scalers/aws/aws_sigv4.go b/pkg/scalers/aws/aws_sigv4.go index bbe18f39dd9..a3a8e45bcd1 100644 --- a/pkg/scalers/aws/aws_sigv4.go +++ b/pkg/scalers/aws/aws_sigv4.go @@ -40,6 +40,7 @@ import ( // roundTripper adds custom round tripper to sign requests type roundTripper struct { client *amp.Client + next http.RoundTripper } var ( @@ -63,11 +64,7 @@ func (rt *roundTripper) RoundTrip(req *http.Request) (*http.Response, error) { if err != nil { return nil, err } - // Create default transport - transport := httputils.CreateHTTPTransport(false) - - // Send signed request - return transport.RoundTrip(req) + return rt.next.RoundTrip(req) } // parseAwsAMPMetadata parses the data to get the AWS specific auth info and metadata @@ -104,6 +101,7 @@ func NewSigV4RoundTripper(config *scalersconfig.ScalerConfig, awsRegion string) client := amp.NewFromConfig(*awsCfg, func(_ *amp.Options) {}) rt := &roundTripper{ client: client, + next: httputils.CreateHTTPTransport(false), } return rt, nil From 011208c769105c96115e6f94a81fedee49ecb4f6 Mon Sep 17 00:00:00 2001 From: Ali Aqel Date: Sun, 12 Apr 2026 23:53:38 +0000 Subject: [PATCH 03/22] fix(scaling): use fresh context with correct scaler labels for retry after refresh Signed-off-by: Ali Aqel --- pkg/scaling/cache/scalers_cache.go | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/pkg/scaling/cache/scalers_cache.go b/pkg/scaling/cache/scalers_cache.go index 1054d29819c..065676f2312 100644 --- a/pkg/scaling/cache/scalers_cache.go +++ b/pkg/scaling/cache/scalers_cache.go @@ -150,6 +150,15 @@ func (c *ScalersCache) GetMetricSpecForScalingForScaler(ctx context.Context, ind return metricSpecs, err } +func buildScalerRequestCtx(ctx context.Context, sb ScalerBuilder, metricName string) context.Context { + requestCtx := context.WithValue(ctx, kedautil.ScalerContextKey, sb.ScalerConfig.TriggerType) + requestCtx = context.WithValue(requestCtx, kedautil.TriggerNameContextKey, sb.ScalerConfig.TriggerName) + requestCtx = context.WithValue(requestCtx, kedautil.MetricNameContextKey, metricName) + requestCtx = context.WithValue(requestCtx, kedautil.NamespaceContextKey, sb.ScalerConfig.ScalableObjectNamespace) + requestCtx = context.WithValue(requestCtx, kedautil.ScaledResourceContextKey, sb.ScalerConfig.ScalableObjectName) + return requestCtx +} + // GetMetricsAndActivityForScaler returns metric value, activity and latency for a scaler identified by the metric name // and by the input index (from the list of scalers in this ScaledObject) func (c *ScalersCache) GetMetricsAndActivityForScaler(ctx context.Context, index int, metricName string) ([]external_metrics.ExternalMetricValue, bool, time.Duration, error) { @@ -157,13 +166,9 @@ func (c *ScalersCache) GetMetricsAndActivityForScaler(ctx context.Context, index if err != nil { return nil, false, -1, err } - ctx = context.WithValue(ctx, kedautil.ScalerContextKey, sb.ScalerConfig.TriggerType) - ctx = context.WithValue(ctx, kedautil.TriggerNameContextKey, sb.ScalerConfig.TriggerName) - ctx = context.WithValue(ctx, kedautil.MetricNameContextKey, metricName) - ctx = context.WithValue(ctx, kedautil.NamespaceContextKey, sb.ScalerConfig.ScalableObjectNamespace) - ctx = context.WithValue(ctx, kedautil.ScaledResourceContextKey, sb.ScalerConfig.ScalableObjectName) + requestCtx := buildScalerRequestCtx(ctx, sb, metricName) startTime := time.Now() - metric, activity, err := sb.Scaler.GetMetricsAndActivity(ctx, metricName) + metric, activity, err := sb.Scaler.GetMetricsAndActivity(requestCtx, metricName) if err == nil { return metric, activity, time.Since(startTime), nil } @@ -172,8 +177,13 @@ func (c *ScalersCache) GetMetricsAndActivityForScaler(ctx context.Context, index if err != nil { return nil, false, -1, err } + newSb, err := c.getScalerBuilder(index) + if err != nil { + return nil, false, -1, err + } + requestCtx = buildScalerRequestCtx(ctx, newSb, metricName) startTime = time.Now() - metric, activity, err = ns.GetMetricsAndActivity(ctx, metricName) + metric, activity, err = ns.GetMetricsAndActivity(requestCtx, metricName) return metric, activity, time.Since(startTime), err } From e7079f089ea18bd6854c8b937451d0a92f1c559b Mon Sep 17 00:00:00 2001 From: Ali Aqel Date: Sun, 12 Apr 2026 23:55:30 +0000 Subject: [PATCH 04/22] refactor(metrics): move httpStatusCodeLabel to metricscollectors.go to remove cross-file coupling Signed-off-by: Ali Aqel --- pkg/metricscollector/metricscollectors.go | 8 ++++++++ pkg/metricscollector/prommetrics.go | 6 ------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/pkg/metricscollector/metricscollectors.go b/pkg/metricscollector/metricscollectors.go index ac6bf7efccf..92df68d3152 100644 --- a/pkg/metricscollector/metricscollectors.go +++ b/pkg/metricscollector/metricscollectors.go @@ -17,6 +17,7 @@ limitations under the License. package metricscollector import ( + "strconv" "time" grpcprom "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus" @@ -219,6 +220,13 @@ func RecordHTTPClientRequest(durationSeconds float64, statusCode int, isError bo } } +func httpStatusCodeLabel(code int, isError bool) string { + if isError { + return "error" + } + return strconv.Itoa(code) +} + // Returns the ServerMetrics object for GRPC Server metrics. Used to initialize the GRPC server with the proper intercepts // Currently, only Prometheus metrics are supported. func GetServerMetrics() *grpcprom.ServerMetrics { diff --git a/pkg/metricscollector/prommetrics.go b/pkg/metricscollector/prommetrics.go index 1dc9d0e3dfc..dad3d3df48d 100644 --- a/pkg/metricscollector/prommetrics.go +++ b/pkg/metricscollector/prommetrics.go @@ -359,12 +359,6 @@ func (p *PromMetrics) RecordHTTPClientRequest(durationSeconds float64, statusCod httpClientRequestDuration.WithLabelValues(namespace, scaledResource, scaler, triggerName, metricName, code).Observe(durationSeconds) } -func httpStatusCodeLabel(code int, isError bool) string { - if isError { - return "error" - } - return strconv.Itoa(code) -} // Returns a grpcprom server Metrics object and registers the metrics. The object contains // interceptors to chain to the server so that all requests served are observed. Intended to be called From 1cacc25ca86b6d3b2bd146ee5edad1082013cd53 Mon Sep 17 00:00:00 2001 From: Ali Aqel Date: Sun, 12 Apr 2026 23:58:25 +0000 Subject: [PATCH 05/22] test: add missing tests for transport instrumentation, status code label precedence, and histogram recording Signed-off-by: Ali Aqel --- pkg/metricscollector/prommetrics_test.go | 32 ++++++++++++++++++------ pkg/util/http_roundtripper_test.go | 6 +++++ 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/pkg/metricscollector/prommetrics_test.go b/pkg/metricscollector/prommetrics_test.go index 478207ba011..576c8799311 100644 --- a/pkg/metricscollector/prommetrics_test.go +++ b/pkg/metricscollector/prommetrics_test.go @@ -19,7 +19,10 @@ package metricscollector import ( "testing" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestHTTPStatusCodeLabel(t *testing.T) { @@ -30,6 +33,7 @@ func TestHTTPStatusCodeLabel(t *testing.T) { want string }{ {"transport error", 0, true, "error"}, + {"isError flag takes precedence over non-zero code", 500, true, "error"}, {"200 OK", 200, false, "200"}, {"201 Created", 201, false, "201"}, {"301 Moved", 301, false, "301"}, @@ -56,19 +60,31 @@ func TestPromMetrics_RecordHTTPClientRequest(t *testing.T) { p.RecordHTTPClientRequest(0.2, 500, false, "prometheus", "my-trigger", "my-metric", "default", "my-so") p.RecordHTTPClientRequest(0.3, 0, true, "", "", "", "", "") + m := &dto.Metric{} + counter, err := httpClientRequestsTotal.GetMetricWithLabelValues("default", "my-so", "prometheus", "my-trigger", "my-metric", "200") - assert.NoError(t, err) - assert.NotNil(t, counter) + require.NoError(t, err) + require.NoError(t, counter.Write(m)) + assert.EqualValues(t, 1, m.Counter.GetValue()) counter, err = httpClientRequestsTotal.GetMetricWithLabelValues("default", "my-so", "redis", "redis-trigger", "redis-metric", "404") - assert.NoError(t, err) - assert.NotNil(t, counter) + require.NoError(t, err) + require.NoError(t, counter.Write(m)) + assert.EqualValues(t, 1, m.Counter.GetValue()) counter, err = httpClientRequestsTotal.GetMetricWithLabelValues("default", "my-so", "prometheus", "my-trigger", "my-metric", "500") - assert.NoError(t, err) - assert.NotNil(t, counter) + require.NoError(t, err) + require.NoError(t, counter.Write(m)) + assert.EqualValues(t, 1, m.Counter.GetValue()) counter, err = httpClientRequestsTotal.GetMetricWithLabelValues("", "", "", "", "", "error") - assert.NoError(t, err) - assert.NotNil(t, counter) + require.NoError(t, err) + require.NoError(t, counter.Write(m)) + assert.EqualValues(t, 1, m.Counter.GetValue()) + + hist, err := httpClientRequestDuration.GetMetricWithLabelValues("default", "my-so", "prometheus", "my-trigger", "my-metric", "200") + require.NoError(t, err) + require.NoError(t, hist.(prometheus.Metric).Write(m)) + assert.EqualValues(t, 1, m.Histogram.GetSampleCount()) + assert.InDelta(t, 0.05, m.Histogram.GetSampleSum(), 0.001) } diff --git a/pkg/util/http_roundtripper_test.go b/pkg/util/http_roundtripper_test.go index d7e3ea971d5..f82cc5e3f75 100644 --- a/pkg/util/http_roundtripper_test.go +++ b/pkg/util/http_roundtripper_test.go @@ -121,3 +121,9 @@ func TestCreateHTTPClient_TransportIsInstrumented(t *testing.T) { _, ok := client.Transport.(*InstrumentedRoundTripper) assert.True(t, ok, "expected CreateHTTPClient to wrap transport with InstrumentedRoundTripper") } + +func TestCreateHTTPTransportWithTLSConfig_IsInstrumented(t *testing.T) { + rt := CreateHTTPTransportWithTLSConfig(nil) + _, ok := rt.(*InstrumentedRoundTripper) + assert.True(t, ok, "expected CreateHTTPTransportWithTLSConfig to return an InstrumentedRoundTripper") +} From a5636e3beece27b0349fd09c0fc0491e11297bea Mon Sep 17 00:00:00 2001 From: Ali Aqel Date: Mon, 13 Apr 2026 00:18:12 +0000 Subject: [PATCH 06/22] test(scaling): verify buildScalerRequestCtx injects correct context key-value pairs Signed-off-by: Ali Aqel --- pkg/scaling/cache/scalers_cache_test.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/pkg/scaling/cache/scalers_cache_test.go b/pkg/scaling/cache/scalers_cache_test.go index 75bccbfcb92..ca1e7fd43f9 100644 --- a/pkg/scaling/cache/scalers_cache_test.go +++ b/pkg/scaling/cache/scalers_cache_test.go @@ -21,8 +21,32 @@ import ( "testing" . "github.com/onsi/gomega" + + kedautil "github.com/kedacore/keda/v2/pkg/util" + "github.com/kedacore/keda/v2/pkg/scalers/scalersconfig" ) +func TestBuildScalerRequestCtx(t *testing.T) { + RegisterTestingT(t) + + sb := ScalerBuilder{ + ScalerConfig: scalersconfig.ScalerConfig{ + TriggerType: "prometheus", + TriggerName: "my-trigger", + ScalableObjectNamespace: "my-namespace", + ScalableObjectName: "my-scaled-object", + }, + } + + ctx := buildScalerRequestCtx(context.Background(), sb, "my-metric") + + Expect(ctx.Value(kedautil.ScalerContextKey)).To(Equal("prometheus")) + Expect(ctx.Value(kedautil.TriggerNameContextKey)).To(Equal("my-trigger")) + Expect(ctx.Value(kedautil.MetricNameContextKey)).To(Equal("my-metric")) + Expect(ctx.Value(kedautil.NamespaceContextKey)).To(Equal("my-namespace")) + Expect(ctx.Value(kedautil.ScaledResourceContextKey)).To(Equal("my-scaled-object")) +} + func TestEmptyScalersCache(t *testing.T) { RegisterTestingT(t) From f00fc014e38d96e9439bd472cac002265f8eb16d Mon Sep 17 00:00:00 2001 From: Ali Aqel Date: Mon, 13 Apr 2026 00:52:46 +0000 Subject: [PATCH 07/22] refactor(metrics): rename to keda_scaler_http_* and only record during metric fetches Signed-off-by: Ali Aqel --- CHANGELOG.md | 2 +- pkg/metricscollector/opentelemetry.go | 8 ++++---- pkg/metricscollector/opentelemetry_test.go | 4 ++-- pkg/metricscollector/prommetrics.go | 8 ++++---- pkg/util/http_roundtripper.go | 18 +++++++++++------- .../opentelemetry_metrics_test.go | 12 ++++++------ .../prometheus_metrics_test.go | 14 +++++++------- 7 files changed, 35 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae2f1d2c7a1..208cfc5daef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,7 +76,7 @@ To learn more about active deprecations, we recommend checking [GitHub Discussio ### Improvements -- **General**: Add HTTP client request metrics for all outbound HTTP requests made by KEDA scalers ([#6600](https://github.com/kedacore/keda/issues/6600)) +- **General**: Add scaler HTTP request metrics (`keda_scaler_http_requests_total`, `keda_scaler_http_request_duration_seconds`) for outbound HTTP requests made during scaler metric fetches ([#6600](https://github.com/kedacore/keda/issues/6600)) - **General**: Add CRD-level validation markers (Minimum, MinLength, MinItems, Enum) for ScaledObject, ScaledJob, ScaleTriggers, and TriggerAuthentication API types ([#7533](https://github.com/kedacore/keda/pull/7533)) - **General**: Add `--leader-election-id` flag to allow configuring the leader election Lease name ([#7564](https://github.com/kedacore/keda/issues/7564)) - **General**: Make APIService cert injections optional ([#7559](https://github.com/kedacore/keda/pull/7559)) diff --git a/pkg/metricscollector/opentelemetry.go b/pkg/metricscollector/opentelemetry.go index 592a2830b2d..c8adef505d2 100644 --- a/pkg/metricscollector/opentelemetry.go +++ b/pkg/metricscollector/opentelemetry.go @@ -225,16 +225,16 @@ func initMeters() { } otHTTPClientRequestsCounter, err = meter.Int64Counter( - "keda.http.client.requests.count", - api.WithDescription("Total number of outbound HTTP requests issued by KEDA's HTTP clients, labeled by status class."), + "keda.scaler.http.requests.count", + api.WithDescription("Total number of outbound HTTP requests issued during scaler metric fetches, labeled by status class."), ) if err != nil { otLog.Error(err, msg) } otHTTPClientRequestDuration, err = meter.Float64Histogram( - "keda.http.client.request.duration.seconds", - api.WithDescription("Duration in seconds of outbound HTTP requests issued by KEDA's HTTP clients."), + "keda.scaler.http.request.duration.seconds", + api.WithDescription("Duration in seconds of outbound HTTP requests issued during scaler metric fetches."), api.WithUnit("s"), ) if err != nil { diff --git a/pkg/metricscollector/opentelemetry_test.go b/pkg/metricscollector/opentelemetry_test.go index 4e7d98c2bbe..a15c1c49973 100644 --- a/pkg/metricscollector/opentelemetry_test.go +++ b/pkg/metricscollector/opentelemetry_test.go @@ -125,7 +125,7 @@ func TestRecordHTTPClientRequest(t *testing.T) { assert.Nil(t, err) scopeMetrics := got.ScopeMetrics[0] - requestCount := retrieveMetric(scopeMetrics.Metrics, "keda.http.client.requests.count") + requestCount := retrieveMetric(scopeMetrics.Metrics, "keda.scaler.http.requests.count") assert.NotNil(t, requestCount) var found bool @@ -149,7 +149,7 @@ func TestRecordHTTPClientRequest(t *testing.T) { } assert.True(t, found, "expected data point with status_code=%q", tt.wantStatusCode) - requestDuration := retrieveMetric(scopeMetrics.Metrics, "keda.http.client.request.duration.seconds") + requestDuration := retrieveMetric(scopeMetrics.Metrics, "keda.scaler.http.request.duration.seconds") assert.NotNil(t, requestDuration) assert.Equal(t, "s", requestDuration.Unit) }) diff --git a/pkg/metricscollector/prommetrics.go b/pkg/metricscollector/prommetrics.go index dad3d3df48d..7dfcd7496e8 100644 --- a/pkg/metricscollector/prommetrics.go +++ b/pkg/metricscollector/prommetrics.go @@ -156,9 +156,9 @@ var ( httpClientRequestsTotal = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: DefaultPromMetricsNamespace, - Subsystem: "http_client", + Subsystem: "scaler_http", Name: "requests_total", - Help: "Total number of outbound HTTP requests issued by KEDA's HTTP clients.", + Help: "Total number of outbound HTTP requests issued during scaler metric fetches.", }, []string{"namespace", "scaled_resource", "scaler", "trigger_name", "metric_name", "status_code"}, ) @@ -166,9 +166,9 @@ var ( httpClientRequestDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: DefaultPromMetricsNamespace, - Subsystem: "http_client", + Subsystem: "scaler_http", Name: "request_duration_seconds", - Help: "Duration in seconds of outbound HTTP requests issued by KEDA's HTTP clients.", + Help: "Duration in seconds of outbound HTTP requests issued during scaler metric fetches.", Buckets: []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10}, }, []string{"namespace", "scaled_resource", "scaler", "trigger_name", "metric_name", "status_code"}, diff --git a/pkg/util/http_roundtripper.go b/pkg/util/http_roundtripper.go index 6451e7d85d7..2a68dd6c558 100644 --- a/pkg/util/http_roundtripper.go +++ b/pkg/util/http_roundtripper.go @@ -74,14 +74,18 @@ func (r *InstrumentedRoundTripper) RoundTrip(req *http.Request) (*http.Response, duration := time.Since(start).Seconds() ctx := req.Context() - scaler, ok := ctx.Value(ScalerContextKey).(string) - if !ok || scaler == "" { - scaler = "unknown" + scaler, scalerOK := ctx.Value(ScalerContextKey).(string) + triggerName, triggerOK := ctx.Value(TriggerNameContextKey).(string) + metricName, metricOK := ctx.Value(MetricNameContextKey).(string) + namespace, nsOK := ctx.Value(NamespaceContextKey).(string) + scaledResource, srOK := ctx.Value(ScaledResourceContextKey).(string) + + // Only record metrics for scaler metric-fetch requests, identified by the + // presence of all five context keys injected by buildScalerRequestCtx. + // Other HTTP calls (e.g. during scaler initialization) are not recorded. + if !scalerOK || !triggerOK || !metricOK || !nsOK || !srOK { + return resp, err } - triggerName, _ := ctx.Value(TriggerNameContextKey).(string) - metricName, _ := ctx.Value(MetricNameContextKey).(string) - namespace, _ := ctx.Value(NamespaceContextKey).(string) - scaledResource, _ := ctx.Value(ScaledResourceContextKey).(string) if err != nil { metricscollector.RecordHTTPClientRequest(duration, 0, true, scaler, triggerName, metricName, namespace, scaledResource) diff --git a/tests/sequential/opentelemetry_metrics/opentelemetry_metrics_test.go b/tests/sequential/opentelemetry_metrics/opentelemetry_metrics_test.go index cacd712ff8a..1ea914645c2 100644 --- a/tests/sequential/opentelemetry_metrics/opentelemetry_metrics_test.go +++ b/tests/sequential/opentelemetry_metrics/opentelemetry_metrics_test.go @@ -1290,8 +1290,8 @@ func testHTTPClientMetrics(t *testing.T, data templateData) { ExtractPrometheusLabelValue("metric_name", labels) == "s0-prometheus" } - val, ok := family["keda_http_client_requests_count_total"] - assert.True(t, ok, "keda_http_client_requests_count_total not available") + val, ok := family["keda_scaler_http_requests_count_total"] + assert.True(t, ok, "keda_scaler_http_requests_count_total not available") if ok { var found bool for _, metric := range val.GetMetric() { @@ -1302,20 +1302,20 @@ func testHTTPClientMetrics(t *testing.T, data templateData) { } } assert.True(t, found, - "expected keda_http_client_requests_count_total with namespace=%s, scaled_resource=%s, scaler=prometheus, trigger_name=%s, metric_name=s0-prometheus", + "expected keda_scaler_http_requests_count_total with namespace=%s, scaled_resource=%s, scaler=prometheus, trigger_name=%s, metric_name=s0-prometheus", data.TestNamespace, wrongScaledObjectName, wrongScalerName) } - if val, ok := family["keda_http_client_request_duration_seconds"]; ok { + if val, ok := family["keda_scaler_http_request_duration_seconds"]; ok { var found bool for _, metric := range val.GetMetric() { if matchLabels(metric.GetLabel()) { assert.Greater(t, metric.GetHistogram().GetSampleCount(), uint64(0), - "keda_http_client_request_duration_seconds sample count should be > 0") + "keda_scaler_http_request_duration_seconds sample count should be > 0") found = true break } } - assert.True(t, found, "expected keda_http_client_request_duration_seconds histogram for prometheus scaler") + assert.True(t, found, "expected keda_scaler_http_request_duration_seconds histogram for prometheus scaler") } } diff --git a/tests/sequential/prometheus_metrics/prometheus_metrics_test.go b/tests/sequential/prometheus_metrics/prometheus_metrics_test.go index e68c8862fc5..c61eb538a79 100644 --- a/tests/sequential/prometheus_metrics/prometheus_metrics_test.go +++ b/tests/sequential/prometheus_metrics/prometheus_metrics_test.go @@ -1462,23 +1462,23 @@ func testHTTPClientMetrics(t *testing.T, data templateData) { return false } - families := WaitForPrometheusMetric(t, "keda_http_client_requests_total", familyValidator) - assert.True(t, familyValidator(families["keda_http_client_requests_total"]), - "expected keda_http_client_requests_total with namespace=%s, scaled_resource=%s, scaler=prometheus, trigger_name=%s, metric_name=s0-prometheus", + families := WaitForPrometheusMetric(t, "keda_scaler_http_requests_total", familyValidator) + assert.True(t, familyValidator(families["keda_scaler_http_requests_total"]), + "expected keda_scaler_http_requests_total with namespace=%s, scaled_resource=%s, scaler=prometheus, trigger_name=%s, metric_name=s0-prometheus", testNamespace, wrongScaledObjectName, wrongScalerName) - family, ok := families["keda_http_client_request_duration_seconds"] - assert.True(t, ok, "keda_http_client_request_duration_seconds not present") + family, ok := families["keda_scaler_http_request_duration_seconds"] + assert.True(t, ok, "keda_scaler_http_request_duration_seconds not present") if ok { var found bool for _, metric := range family.GetMetric() { if matchLabels(metric.GetLabel()) { assert.Greater(t, metric.GetHistogram().GetSampleCount(), uint64(0), - "keda_http_client_request_duration_seconds sample count should be > 0") + "keda_scaler_http_request_duration_seconds sample count should be > 0") found = true break } } - assert.True(t, found, "expected keda_http_client_request_duration_seconds histogram for prometheus scaler") + assert.True(t, found, "expected keda_scaler_http_request_duration_seconds histogram for prometheus scaler") } } From efa0a76cd1019e41bdefdedd1fc07e9247935a23 Mon Sep 17 00:00:00 2001 From: Ali Aqel Date: Mon, 13 Apr 2026 01:10:22 +0000 Subject: [PATCH 08/22] fix(metrics): use accurate log message for OTel instrument registration failures Signed-off-by: Ali Aqel --- pkg/metricscollector/opentelemetry.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/metricscollector/opentelemetry.go b/pkg/metricscollector/opentelemetry.go index c8adef505d2..b56f4faf62d 100644 --- a/pkg/metricscollector/opentelemetry.go +++ b/pkg/metricscollector/opentelemetry.go @@ -101,7 +101,7 @@ func NewOtelMetrics(options ...metric.Option) *OtelMetrics { func initMeters() { var err error - msg := "create opentelemetry counter failed" + msg := "failed to create OpenTelemetry instrument" otScalerErrorsCounter, err = meter.Int64Counter("keda.scaler.errors", api.WithDescription("Number of scaler errors")) if err != nil { From a786ee7dd19181567e0c34d2136436cff1a5ffff Mon Sep 17 00:00:00 2001 From: Ali Aqel Date: Mon, 13 Apr 2026 01:41:22 +0000 Subject: [PATCH 09/22] chore: update copyright year to 2026 on new files Signed-off-by: Ali Aqel --- pkg/metricscollector/prommetrics_test.go | 2 +- pkg/util/http_roundtripper.go | 2 +- pkg/util/http_roundtripper_test.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/metricscollector/prommetrics_test.go b/pkg/metricscollector/prommetrics_test.go index 576c8799311..443edaf10dc 100644 --- a/pkg/metricscollector/prommetrics_test.go +++ b/pkg/metricscollector/prommetrics_test.go @@ -1,5 +1,5 @@ /* -Copyright 2025 The KEDA Authors +Copyright 2026 The KEDA Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/util/http_roundtripper.go b/pkg/util/http_roundtripper.go index 2a68dd6c558..bb132067a40 100644 --- a/pkg/util/http_roundtripper.go +++ b/pkg/util/http_roundtripper.go @@ -1,5 +1,5 @@ /* -Copyright 2025 The KEDA Authors +Copyright 2026 The KEDA Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/util/http_roundtripper_test.go b/pkg/util/http_roundtripper_test.go index f82cc5e3f75..0d3d61f743b 100644 --- a/pkg/util/http_roundtripper_test.go +++ b/pkg/util/http_roundtripper_test.go @@ -1,5 +1,5 @@ /* -Copyright 2025 The KEDA Authors +Copyright 2026 The KEDA Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. From 90c3e85f9580d114ac2f43d530fe878fd1bdb3e3 Mon Sep 17 00:00:00 2001 From: Ali Aqel Date: Mon, 13 Apr 2026 01:44:32 +0000 Subject: [PATCH 10/22] chore: use "scaler metric collection" instead of "scaler metric fetch" Signed-off-by: Ali Aqel --- CHANGELOG.md | 2 +- pkg/metricscollector/opentelemetry.go | 4 ++-- pkg/metricscollector/prommetrics.go | 4 ++-- pkg/util/http_roundtripper.go | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 208cfc5daef..4888fb7cbe7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,7 +76,7 @@ To learn more about active deprecations, we recommend checking [GitHub Discussio ### Improvements -- **General**: Add scaler HTTP request metrics (`keda_scaler_http_requests_total`, `keda_scaler_http_request_duration_seconds`) for outbound HTTP requests made during scaler metric fetches ([#6600](https://github.com/kedacore/keda/issues/6600)) +- **General**: Add scaler HTTP request metrics (`keda_scaler_http_requests_total`, `keda_scaler_http_request_duration_seconds`) for outbound HTTP requests made during scaler metric collection ([#6600](https://github.com/kedacore/keda/issues/6600)) - **General**: Add CRD-level validation markers (Minimum, MinLength, MinItems, Enum) for ScaledObject, ScaledJob, ScaleTriggers, and TriggerAuthentication API types ([#7533](https://github.com/kedacore/keda/pull/7533)) - **General**: Add `--leader-election-id` flag to allow configuring the leader election Lease name ([#7564](https://github.com/kedacore/keda/issues/7564)) - **General**: Make APIService cert injections optional ([#7559](https://github.com/kedacore/keda/pull/7559)) diff --git a/pkg/metricscollector/opentelemetry.go b/pkg/metricscollector/opentelemetry.go index b56f4faf62d..299ab9050ef 100644 --- a/pkg/metricscollector/opentelemetry.go +++ b/pkg/metricscollector/opentelemetry.go @@ -226,7 +226,7 @@ func initMeters() { otHTTPClientRequestsCounter, err = meter.Int64Counter( "keda.scaler.http.requests.count", - api.WithDescription("Total number of outbound HTTP requests issued during scaler metric fetches, labeled by status class."), + api.WithDescription("Total number of outbound HTTP requests issued during scaler metric collection, labeled by status class."), ) if err != nil { otLog.Error(err, msg) @@ -234,7 +234,7 @@ func initMeters() { otHTTPClientRequestDuration, err = meter.Float64Histogram( "keda.scaler.http.request.duration.seconds", - api.WithDescription("Duration in seconds of outbound HTTP requests issued during scaler metric fetches."), + api.WithDescription("Duration in seconds of outbound HTTP requests issued during scaler metric collection."), api.WithUnit("s"), ) if err != nil { diff --git a/pkg/metricscollector/prommetrics.go b/pkg/metricscollector/prommetrics.go index 7dfcd7496e8..c8d8120adaa 100644 --- a/pkg/metricscollector/prommetrics.go +++ b/pkg/metricscollector/prommetrics.go @@ -158,7 +158,7 @@ var ( Namespace: DefaultPromMetricsNamespace, Subsystem: "scaler_http", Name: "requests_total", - Help: "Total number of outbound HTTP requests issued during scaler metric fetches.", + Help: "Total number of outbound HTTP requests issued during scaler metric collection.", }, []string{"namespace", "scaled_resource", "scaler", "trigger_name", "metric_name", "status_code"}, ) @@ -168,7 +168,7 @@ var ( Namespace: DefaultPromMetricsNamespace, Subsystem: "scaler_http", Name: "request_duration_seconds", - Help: "Duration in seconds of outbound HTTP requests issued during scaler metric fetches.", + Help: "Duration in seconds of outbound HTTP requests issued during scaler metric collection.", Buckets: []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10}, }, []string{"namespace", "scaled_resource", "scaler", "trigger_name", "metric_name", "status_code"}, diff --git a/pkg/util/http_roundtripper.go b/pkg/util/http_roundtripper.go index bb132067a40..4af4d4a283d 100644 --- a/pkg/util/http_roundtripper.go +++ b/pkg/util/http_roundtripper.go @@ -80,7 +80,7 @@ func (r *InstrumentedRoundTripper) RoundTrip(req *http.Request) (*http.Response, namespace, nsOK := ctx.Value(NamespaceContextKey).(string) scaledResource, srOK := ctx.Value(ScaledResourceContextKey).(string) - // Only record metrics for scaler metric-fetch requests, identified by the + // Only record metrics for scaler metric collection requests, identified by the // presence of all five context keys injected by buildScalerRequestCtx. // Other HTTP calls (e.g. during scaler initialization) are not recorded. if !scalerOK || !triggerOK || !metricOK || !nsOK || !srOK { From 0e9bdc578135ebad93351364047267344a7fc24c Mon Sep 17 00:00:00 2001 From: Ali Aqel Date: Mon, 13 Apr 2026 15:12:26 +0000 Subject: [PATCH 11/22] feat: reduce histogram label cardinality to scaler + status_code The duration histogram keda_scaler_http_request_duration_seconds previously carried 6 label dimensions (namespace, scaled_resource, scaler, trigger_name, metric_name, status_code), creating high MTS cardinality. Latency by scaler type is what matters, so the histogram is reduced to 2 labels: scaler and status_code. The counter retains all 6 labels. Signed-off-by: Ali Aqel --- pkg/metricscollector/opentelemetry.go | 10 +++++++--- pkg/metricscollector/prommetrics.go | 4 ++-- pkg/metricscollector/prommetrics_test.go | 2 +- .../opentelemetry_metrics_test.go | 5 ++++- .../prometheus_metrics/prometheus_metrics_test.go | 5 ++++- 5 files changed, 18 insertions(+), 8 deletions(-) diff --git a/pkg/metricscollector/opentelemetry.go b/pkg/metricscollector/opentelemetry.go index 299ab9050ef..1ab52d96615 100644 --- a/pkg/metricscollector/opentelemetry.go +++ b/pkg/metricscollector/opentelemetry.go @@ -518,7 +518,7 @@ func CloudeventQueueStatusCallback(_ context.Context, obsrv api.Float64Observer) // RecordHTTPClientRequest records the duration and outcome of a single outbound HTTP request. func (o *OtelMetrics) RecordHTTPClientRequest(durationSeconds float64, statusCode int, isError bool, scaler, triggerName, metricName, namespace, scaledResource string) { code := httpStatusCodeLabel(statusCode, isError) - opt := api.WithAttributes( + counterOpt := api.WithAttributes( attribute.Key("namespace").String(namespace), attribute.Key("scaled_resource").String(scaledResource), attribute.Key("scaler").String(scaler), @@ -526,8 +526,12 @@ func (o *OtelMetrics) RecordHTTPClientRequest(durationSeconds float64, statusCod attribute.Key("metric_name").String(metricName), attribute.Key("status_code").String(code), ) - otHTTPClientRequestsCounter.Add(context.Background(), 1, opt) - otHTTPClientRequestDuration.Record(context.Background(), durationSeconds, opt) + histOpt := api.WithAttributes( + attribute.Key("scaler").String(scaler), + attribute.Key("status_code").String(code), + ) + otHTTPClientRequestsCounter.Add(context.Background(), 1, counterOpt) + otHTTPClientRequestDuration.Record(context.Background(), durationSeconds, histOpt) } // RecordCloudEventQueueStatus record the number of cloudevents that are waiting for emitting diff --git a/pkg/metricscollector/prommetrics.go b/pkg/metricscollector/prommetrics.go index c8d8120adaa..86bfc3bf3ac 100644 --- a/pkg/metricscollector/prommetrics.go +++ b/pkg/metricscollector/prommetrics.go @@ -171,7 +171,7 @@ var ( Help: "Duration in seconds of outbound HTTP requests issued during scaler metric collection.", Buckets: []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10}, }, - []string{"namespace", "scaled_resource", "scaler", "trigger_name", "metric_name", "status_code"}, + []string{"scaler", "status_code"}, ) ) @@ -356,7 +356,7 @@ func (p *PromMetrics) RecordCloudEventQueueStatus(namespace string, value int) { func (p *PromMetrics) RecordHTTPClientRequest(durationSeconds float64, statusCode int, isError bool, scaler, triggerName, metricName, namespace, scaledResource string) { code := httpStatusCodeLabel(statusCode, isError) httpClientRequestsTotal.WithLabelValues(namespace, scaledResource, scaler, triggerName, metricName, code).Inc() - httpClientRequestDuration.WithLabelValues(namespace, scaledResource, scaler, triggerName, metricName, code).Observe(durationSeconds) + httpClientRequestDuration.WithLabelValues(scaler, code).Observe(durationSeconds) } diff --git a/pkg/metricscollector/prommetrics_test.go b/pkg/metricscollector/prommetrics_test.go index 443edaf10dc..eb4bb6fc153 100644 --- a/pkg/metricscollector/prommetrics_test.go +++ b/pkg/metricscollector/prommetrics_test.go @@ -82,7 +82,7 @@ func TestPromMetrics_RecordHTTPClientRequest(t *testing.T) { require.NoError(t, counter.Write(m)) assert.EqualValues(t, 1, m.Counter.GetValue()) - hist, err := httpClientRequestDuration.GetMetricWithLabelValues("default", "my-so", "prometheus", "my-trigger", "my-metric", "200") + hist, err := httpClientRequestDuration.GetMetricWithLabelValues("prometheus", "200") require.NoError(t, err) require.NoError(t, hist.(prometheus.Metric).Write(m)) assert.EqualValues(t, 1, m.Histogram.GetSampleCount()) diff --git a/tests/sequential/opentelemetry_metrics/opentelemetry_metrics_test.go b/tests/sequential/opentelemetry_metrics/opentelemetry_metrics_test.go index 1ea914645c2..83bf7e6eb3e 100644 --- a/tests/sequential/opentelemetry_metrics/opentelemetry_metrics_test.go +++ b/tests/sequential/opentelemetry_metrics/opentelemetry_metrics_test.go @@ -1306,10 +1306,13 @@ func testHTTPClientMetrics(t *testing.T, data templateData) { data.TestNamespace, wrongScaledObjectName, wrongScalerName) } + matchHistogramLabels := func(labels []*prommodel.LabelPair) bool { + return ExtractPrometheusLabelValue("scaler", labels) == "prometheus" + } if val, ok := family["keda_scaler_http_request_duration_seconds"]; ok { var found bool for _, metric := range val.GetMetric() { - if matchLabels(metric.GetLabel()) { + if matchHistogramLabels(metric.GetLabel()) { assert.Greater(t, metric.GetHistogram().GetSampleCount(), uint64(0), "keda_scaler_http_request_duration_seconds sample count should be > 0") found = true diff --git a/tests/sequential/prometheus_metrics/prometheus_metrics_test.go b/tests/sequential/prometheus_metrics/prometheus_metrics_test.go index c61eb538a79..212615b23b5 100644 --- a/tests/sequential/prometheus_metrics/prometheus_metrics_test.go +++ b/tests/sequential/prometheus_metrics/prometheus_metrics_test.go @@ -1467,12 +1467,15 @@ func testHTTPClientMetrics(t *testing.T, data templateData) { "expected keda_scaler_http_requests_total with namespace=%s, scaled_resource=%s, scaler=prometheus, trigger_name=%s, metric_name=s0-prometheus", testNamespace, wrongScaledObjectName, wrongScalerName) + matchHistogramLabels := func(labels []*prommodel.LabelPair) bool { + return ExtractPrometheusLabelValue("scaler", labels) == "prometheus" + } family, ok := families["keda_scaler_http_request_duration_seconds"] assert.True(t, ok, "keda_scaler_http_request_duration_seconds not present") if ok { var found bool for _, metric := range family.GetMetric() { - if matchLabels(metric.GetLabel()) { + if matchHistogramLabels(metric.GetLabel()) { assert.Greater(t, metric.GetHistogram().GetSampleCount(), uint64(0), "keda_scaler_http_request_duration_seconds sample count should be > 0") found = true From 9c79fcedce35cc108090a2a218b2015485b048fd Mon Sep 17 00:00:00 2001 From: Ali Aqel Date: Mon, 13 Apr 2026 15:46:37 +0000 Subject: [PATCH 12/22] fix: gofmt and CHANGELOG sort order for static checks Signed-off-by: Ali Aqel --- CHANGELOG.md | 2 +- pkg/metricscollector/prommetrics.go | 1 - pkg/scaling/cache/scalers_cache_test.go | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4888fb7cbe7..98bd78f1877 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,9 +76,9 @@ To learn more about active deprecations, we recommend checking [GitHub Discussio ### Improvements -- **General**: Add scaler HTTP request metrics (`keda_scaler_http_requests_total`, `keda_scaler_http_request_duration_seconds`) for outbound HTTP requests made during scaler metric collection ([#6600](https://github.com/kedacore/keda/issues/6600)) - **General**: Add CRD-level validation markers (Minimum, MinLength, MinItems, Enum) for ScaledObject, ScaledJob, ScaleTriggers, and TriggerAuthentication API types ([#7533](https://github.com/kedacore/keda/pull/7533)) - **General**: Add `--leader-election-id` flag to allow configuring the leader election Lease name ([#7564](https://github.com/kedacore/keda/issues/7564)) +- **General**: Add scaler HTTP request metrics (`keda_scaler_http_requests_total`, `keda_scaler_http_request_duration_seconds`) for outbound HTTP requests made during scaler metric collection ([#6600](https://github.com/kedacore/keda/issues/6600)) - **General**: Make APIService cert injections optional ([#7559](https://github.com/kedacore/keda/pull/7559)) - **Elasticsearch Scaler**: Add HTTP status check for Elasticsearch errors ([#7480](https://github.com/kedacore/keda/pull/7480)) - **Kubernetes Workload Scaler**: Add `groupByNode` parameter ([#7628](https://github.com/kedacore/keda/issues/7628)) diff --git a/pkg/metricscollector/prommetrics.go b/pkg/metricscollector/prommetrics.go index 86bfc3bf3ac..06f5da5cb1f 100644 --- a/pkg/metricscollector/prommetrics.go +++ b/pkg/metricscollector/prommetrics.go @@ -359,7 +359,6 @@ func (p *PromMetrics) RecordHTTPClientRequest(durationSeconds float64, statusCod httpClientRequestDuration.WithLabelValues(scaler, code).Observe(durationSeconds) } - // Returns a grpcprom server Metrics object and registers the metrics. The object contains // interceptors to chain to the server so that all requests served are observed. Intended to be called // as part of initialization of metricscollector, hence why this function is not exported diff --git a/pkg/scaling/cache/scalers_cache_test.go b/pkg/scaling/cache/scalers_cache_test.go index ca1e7fd43f9..268484448bc 100644 --- a/pkg/scaling/cache/scalers_cache_test.go +++ b/pkg/scaling/cache/scalers_cache_test.go @@ -22,8 +22,8 @@ import ( . "github.com/onsi/gomega" - kedautil "github.com/kedacore/keda/v2/pkg/util" "github.com/kedacore/keda/v2/pkg/scalers/scalersconfig" + kedautil "github.com/kedacore/keda/v2/pkg/util" ) func TestBuildScalerRequestCtx(t *testing.T) { From 5c231493808f4338c7a912b2e942b408ff5236ef Mon Sep 17 00:00:00 2001 From: Ali Aqel Date: Mon, 13 Apr 2026 17:34:19 +0000 Subject: [PATCH 13/22] ci: retrigger static checks Signed-off-by: Ali Aqel From 216420331ece84cb7e06beaa5012862231afd4f2 Mon Sep 17 00:00:00 2001 From: Ali Aqel Date: Wed, 22 Apr 2026 03:59:19 +0000 Subject: [PATCH 14/22] fix: address PR review feedback on HTTP client metrics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rename CreateHTTPTransport/CreateHTTPTransportWithTLSConfig to CreateRT/CreateRTWithTLSConfig to reflect they return http.RoundTripper - Fix OTel metric description: "labeled by status class" → "labeled by HTTP status code" - Fix doc comments on RecordHTTPClientRequest to accurately describe that context keys are extracted by InstrumentedRoundTripper, not the collector - Fix test comments and rename TestInstrumentedRoundTripper_ScalerContextKey to TestInstrumentedRoundTripper_AllContextKeys with all five required context keys set Signed-off-by: Ali Aqel --- pkg/metricscollector/metricscollectors.go | 9 ++++---- pkg/metricscollector/opentelemetry.go | 2 +- pkg/scalers/artemis_scaler.go | 2 +- pkg/scalers/aws/aws_sigv4.go | 2 +- pkg/scalers/aws/aws_sigv4_test.go | 2 +- ...e_managed_prometheus_http_round_tripper.go | 2 +- pkg/scalers/elasticsearch_scaler.go | 2 +- pkg/scalers/github_runner_scaler.go | 2 +- pkg/scalers/ibmmq_scaler.go | 2 +- pkg/scalers/metrics_api_scaler.go | 2 +- pkg/scalers/pulsar_scaler.go | 2 +- pkg/scalers/rabbitmq_scaler.go | 2 +- pkg/util/http.go | 16 ++++++------- pkg/util/http_roundtripper_test.go | 23 +++++++++++-------- 14 files changed, 38 insertions(+), 32 deletions(-) diff --git a/pkg/metricscollector/metricscollectors.go b/pkg/metricscollector/metricscollectors.go index 92df68d3152..c3f62f768fc 100644 --- a/pkg/metricscollector/metricscollectors.go +++ b/pkg/metricscollector/metricscollectors.go @@ -82,8 +82,9 @@ type MetricsCollector interface { RecordCloudEventQueueStatus(namespace string, value int) // RecordHTTPClientRequest records the duration and outcome of an outbound HTTP request - // made by one of KEDA's internal HTTP clients. scaler, triggerName, and metricName - // are read from context keys set in util package; empty string if unset. + // made by one of KEDA's internal HTTP clients. The scaler, triggerName, metricName, + // namespace, and scaledResource values are extracted from context keys by + // InstrumentedRoundTripper in the util package before this method is called. RecordHTTPClientRequest(durationSeconds float64, statusCode int, isError bool, scaler, triggerName, metricName, namespace, scaledResource string) } @@ -212,8 +213,8 @@ func RecordCloudEventQueueStatus(namespace string, value int) { } // RecordHTTPClientRequest records the duration and outcome of an outbound HTTP request -// made by one of KEDA's internal HTTP clients. scaler, triggerName, and metricName -// are read from context keys set in util package; empty string if unset. +// made by one of KEDA's internal HTTP clients. Called by InstrumentedRoundTripper in +// the util package after extracting label values from the request context. func RecordHTTPClientRequest(durationSeconds float64, statusCode int, isError bool, scaler, triggerName, metricName, namespace, scaledResource string) { for _, element := range collectors { element.RecordHTTPClientRequest(durationSeconds, statusCode, isError, scaler, triggerName, metricName, namespace, scaledResource) diff --git a/pkg/metricscollector/opentelemetry.go b/pkg/metricscollector/opentelemetry.go index 1ab52d96615..f9635a4b2c8 100644 --- a/pkg/metricscollector/opentelemetry.go +++ b/pkg/metricscollector/opentelemetry.go @@ -226,7 +226,7 @@ func initMeters() { otHTTPClientRequestsCounter, err = meter.Int64Counter( "keda.scaler.http.requests.count", - api.WithDescription("Total number of outbound HTTP requests issued during scaler metric collection, labeled by status class."), + api.WithDescription("Total number of outbound HTTP requests issued during scaler metric collection, labeled by HTTP status code."), ) if err != nil { otLog.Error(err, msg) diff --git a/pkg/scalers/artemis_scaler.go b/pkg/scalers/artemis_scaler.go index f031f54efac..89d425b0c87 100644 --- a/pkg/scalers/artemis_scaler.go +++ b/pkg/scalers/artemis_scaler.go @@ -109,7 +109,7 @@ func NewArtemisQueueScaler(config *scalersconfig.ScalerConfig) (Scaler, error) { if err != nil { return nil, fmt.Errorf("error creating TLS config: %w", err) } - httpClient.Transport = kedautil.CreateHTTPTransportWithTLSConfig(tlsConfig) + httpClient.Transport = kedautil.CreateRTWithTLSConfig(tlsConfig) } return &artemisScaler{ diff --git a/pkg/scalers/aws/aws_sigv4.go b/pkg/scalers/aws/aws_sigv4.go index a3a8e45bcd1..6f5eb500c01 100644 --- a/pkg/scalers/aws/aws_sigv4.go +++ b/pkg/scalers/aws/aws_sigv4.go @@ -101,7 +101,7 @@ func NewSigV4RoundTripper(config *scalersconfig.ScalerConfig, awsRegion string) client := amp.NewFromConfig(*awsCfg, func(_ *amp.Options) {}) rt := &roundTripper{ client: client, - next: httputils.CreateHTTPTransport(false), + next: httputils.CreateRT(false), } return rt, nil diff --git a/pkg/scalers/aws/aws_sigv4_test.go b/pkg/scalers/aws/aws_sigv4_test.go index 8abc170aea0..9c49be40143 100644 --- a/pkg/scalers/aws/aws_sigv4_test.go +++ b/pkg/scalers/aws/aws_sigv4_test.go @@ -36,7 +36,7 @@ import ( ) func TestSigV4RoundTripper(t *testing.T) { - transport := util.CreateHTTPTransport(false) + transport := util.CreateRT(false) cli := &http.Client{Transport: transport} diff --git a/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go index a40d2b2deaa..123d06f47c2 100644 --- a/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go +++ b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go @@ -55,7 +55,7 @@ func TryAndGetAzureManagedPrometheusHTTPRoundTripper(logger logr.Logger, podIden return nil, err } - transport := util.CreateHTTPTransport(false) + transport := util.CreateRT(false) rt := &azureManagedPrometheusHTTPRoundTripper{ next: transport, chainedCredential: chainedCred, diff --git a/pkg/scalers/elasticsearch_scaler.go b/pkg/scalers/elasticsearch_scaler.go index dae001cea5c..2b4aeea876c 100644 --- a/pkg/scalers/elasticsearch_scaler.go +++ b/pkg/scalers/elasticsearch_scaler.go @@ -130,7 +130,7 @@ func newElasticsearchClient(meta elasticsearchMetadata, logger logr.Logger) (*el } } - config.Transport = util.CreateHTTPTransport(meta.UnsafeSsl) + config.Transport = util.CreateRT(meta.UnsafeSsl) esClient, err := elasticsearch.NewClient(config) if err != nil { logger.Error(err, fmt.Sprintf("Found error when creating client: %s", err)) diff --git a/pkg/scalers/github_runner_scaler.go b/pkg/scalers/github_runner_scaler.go index 448c687ec94..5086d44e157 100644 --- a/pkg/scalers/github_runner_scaler.go +++ b/pkg/scalers/github_runner_scaler.go @@ -347,7 +347,7 @@ func NewGitHubRunnerScaler(config *scalersconfig.ScalerConfig) (Scaler, error) { } if meta.ApplicationID != 0 && meta.InstallationID != 0 && meta.ApplicationKey != "" { - httpTrans := kedautil.CreateHTTPTransport(false) + httpTrans := kedautil.CreateRT(false) hc, err := gha.New(httpTrans, meta.ApplicationID, meta.InstallationID, []byte(meta.ApplicationKey)) if err != nil { return nil, fmt.Errorf("error creating GitHub App client: %w, \n appID: %d, instID: %d", err, meta.ApplicationID, meta.InstallationID) diff --git a/pkg/scalers/ibmmq_scaler.go b/pkg/scalers/ibmmq_scaler.go index 9eff4f96bbc..9fc032c9e5b 100644 --- a/pkg/scalers/ibmmq_scaler.go +++ b/pkg/scalers/ibmmq_scaler.go @@ -98,7 +98,7 @@ func NewIBMMQScaler(config *scalersconfig.ScalerConfig) (Scaler, error) { if err != nil { return nil, err } - httpClient.Transport = kedautil.CreateHTTPTransportWithTLSConfig(tlsConfig) + httpClient.Transport = kedautil.CreateRTWithTLSConfig(tlsConfig) } scaler := &ibmmqScaler{ diff --git a/pkg/scalers/metrics_api_scaler.go b/pkg/scalers/metrics_api_scaler.go index 56eb802520c..1e47713aa7f 100644 --- a/pkg/scalers/metrics_api_scaler.go +++ b/pkg/scalers/metrics_api_scaler.go @@ -101,7 +101,7 @@ func NewMetricsAPIScaler(config *scalersconfig.ScalerConfig, kubeClient client.C if err != nil { return nil, err } - httpClient.Transport = kedautil.CreateHTTPTransportWithTLSConfig(tlsConfig) + httpClient.Transport = kedautil.CreateRTWithTLSConfig(tlsConfig) } return &metricsAPIScaler{ diff --git a/pkg/scalers/pulsar_scaler.go b/pkg/scalers/pulsar_scaler.go index 6b8d587cd8e..4062e312c77 100644 --- a/pkg/scalers/pulsar_scaler.go +++ b/pkg/scalers/pulsar_scaler.go @@ -165,7 +165,7 @@ func NewPulsarScaler(config *scalersconfig.ScalerConfig) (Scaler, error) { if err != nil { return nil, err } - client.Transport = kedautil.CreateHTTPTransportWithTLSConfig(tlsConfig) + client.Transport = kedautil.CreateRTWithTLSConfig(tlsConfig) } if pulsarMetadata.PulsarAuth.EnabledBearerAuth() || pulsarMetadata.PulsarAuth.EnabledBasicAuth() { diff --git a/pkg/scalers/rabbitmq_scaler.go b/pkg/scalers/rabbitmq_scaler.go index 643549f2c65..04d666750dd 100644 --- a/pkg/scalers/rabbitmq_scaler.go +++ b/pkg/scalers/rabbitmq_scaler.go @@ -298,7 +298,7 @@ func NewRabbitMQScaler(config *scalersconfig.ScalerConfig) (Scaler, error) { if tlsErr != nil { return nil, tlsErr } - s.httpClient.Transport = kedautil.CreateHTTPTransportWithTLSConfig(tlsConfig) + s.httpClient.Transport = kedautil.CreateRTWithTLSConfig(tlsConfig) } if meta.Protocol == amqpProtocol { diff --git a/pkg/util/http.go b/pkg/util/http.go index adc35c6728f..090d0b26266 100644 --- a/pkg/util/http.go +++ b/pkg/util/http.go @@ -50,7 +50,7 @@ func CreateHTTPClient(timeout time.Duration, unsafeSsl bool) *http.Client { if timeout <= 0 { timeout = 300 * time.Millisecond } - transport := CreateHTTPTransport(unsafeSsl) + transport := CreateRT(unsafeSsl) httpClient := &http.Client{ Timeout: timeout, Transport: transport, @@ -58,15 +58,15 @@ func CreateHTTPClient(timeout time.Duration, unsafeSsl bool) *http.Client { return httpClient } -// CreateHTTPTransport returns a new HTTP Transport with Proxy, Keep alives -// unsafeSsl parameter allows to avoid tls cert validation if it's required -func CreateHTTPTransport(unsafeSsl bool) http.RoundTripper { - return CreateHTTPTransportWithTLSConfig(CreateTLSClientConfig(unsafeSsl)) +// CreateRT returns a new instrumented HTTP RoundTripper with Proxy and Keep-alive settings. +// unsafeSsl parameter allows to avoid tls cert validation if it's required. +func CreateRT(unsafeSsl bool) http.RoundTripper { + return CreateRTWithTLSConfig(CreateTLSClientConfig(unsafeSsl)) } -// CreateHTTPTransportWithTLSConfig returns a new HTTP Transport with Proxy, Keep alives -// using given tls.Config, wrapped with instrumentation for HTTP client metrics. -func CreateHTTPTransportWithTLSConfig(config *tls.Config) http.RoundTripper { +// CreateRTWithTLSConfig returns a new instrumented HTTP RoundTripper with Proxy and Keep-alive +// settings using the given tls.Config. +func CreateRTWithTLSConfig(config *tls.Config) http.RoundTripper { transport := &http.Transport{ TLSClientConfig: config, Proxy: http.ProxyFromEnvironment, diff --git a/pkg/util/http_roundtripper_test.go b/pkg/util/http_roundtripper_test.go index 0d3d61f743b..2fde011feff 100644 --- a/pkg/util/http_roundtripper_test.go +++ b/pkg/util/http_roundtripper_test.go @@ -92,8 +92,8 @@ func TestInstrumentedRoundTripper_NilNextUsesDefault(t *testing.T) { } func TestInstrumentedRoundTripper_ScalerContextKey_Missing(t *testing.T) { - // When ScalerContextKey is absent the RoundTripper should not panic and - // should complete normally — "unknown" is used as the scaler label value. + // When one or more of the five required context keys are absent, the + // RoundTripper should not panic, complete normally, and skip metric recording. rt := NewInstrumentedRoundTripper(&mockRoundTripper{resp: fakeResponse(200)}) resp, err := rt.RoundTrip(newRequest(context.Background())) @@ -103,12 +103,17 @@ func TestInstrumentedRoundTripper_ScalerContextKey_Missing(t *testing.T) { assert.Equal(t, 200, resp.StatusCode) } -func TestInstrumentedRoundTripper_ScalerContextKey(t *testing.T) { - // Requests with ScalerContextKey set should not panic or error — the key - // is read inside RoundTrip and forwarded to the metrics collector. +func TestInstrumentedRoundTripper_AllContextKeys(t *testing.T) { + // When all five required context keys are present the RoundTripper should + // complete normally and forward the request to the underlying transport. rt := NewInstrumentedRoundTripper(&mockRoundTripper{resp: fakeResponse(200)}) - ctx := context.WithValue(context.Background(), ScalerContextKey, "prometheus") + ctx := context.Background() + ctx = context.WithValue(ctx, ScalerContextKey, "prometheus") + ctx = context.WithValue(ctx, TriggerNameContextKey, "my-trigger") + ctx = context.WithValue(ctx, MetricNameContextKey, "my-metric") + ctx = context.WithValue(ctx, NamespaceContextKey, "default") + ctx = context.WithValue(ctx, ScaledResourceContextKey, "my-so") resp, err := rt.RoundTrip(newRequest(ctx)) require.NoError(t, err) @@ -122,8 +127,8 @@ func TestCreateHTTPClient_TransportIsInstrumented(t *testing.T) { assert.True(t, ok, "expected CreateHTTPClient to wrap transport with InstrumentedRoundTripper") } -func TestCreateHTTPTransportWithTLSConfig_IsInstrumented(t *testing.T) { - rt := CreateHTTPTransportWithTLSConfig(nil) +func TestCreateRTWithTLSConfig_IsInstrumented(t *testing.T) { + rt := CreateRTWithTLSConfig(nil) _, ok := rt.(*InstrumentedRoundTripper) - assert.True(t, ok, "expected CreateHTTPTransportWithTLSConfig to return an InstrumentedRoundTripper") + assert.True(t, ok, "expected CreateRTWithTLSConfig to return an InstrumentedRoundTripper") } From aa5a23c01b12201e7cfef123d06ef70c06e41443 Mon Sep 17 00:00:00 2001 From: Ali Aqel Date: Wed, 22 Apr 2026 04:07:04 +0000 Subject: [PATCH 15/22] revert: keep CreateHTTPTransport* names, no rename needed The return type change (*http.Transport -> http.RoundTripper) is non-breaking for all callers since http.Client.Transport and all struct fields receiving the value are already typed as http.RoundTripper. No caller accesses *http.Transport-specific fields, so the rename to CreateRT* is unnecessary churn. Signed-off-by: Ali Aqel --- pkg/scalers/artemis_scaler.go | 2 +- pkg/scalers/aws/aws_sigv4.go | 2 +- pkg/scalers/aws/aws_sigv4_test.go | 2 +- .../azure_managed_prometheus_http_round_tripper.go | 2 +- pkg/scalers/elasticsearch_scaler.go | 2 +- pkg/scalers/github_runner_scaler.go | 2 +- pkg/scalers/ibmmq_scaler.go | 2 +- pkg/scalers/metrics_api_scaler.go | 2 +- pkg/scalers/pulsar_scaler.go | 2 +- pkg/scalers/rabbitmq_scaler.go | 2 +- pkg/util/http.go | 14 +++++++------- pkg/util/http_roundtripper_test.go | 6 +++--- 12 files changed, 20 insertions(+), 20 deletions(-) diff --git a/pkg/scalers/artemis_scaler.go b/pkg/scalers/artemis_scaler.go index 89d425b0c87..f031f54efac 100644 --- a/pkg/scalers/artemis_scaler.go +++ b/pkg/scalers/artemis_scaler.go @@ -109,7 +109,7 @@ func NewArtemisQueueScaler(config *scalersconfig.ScalerConfig) (Scaler, error) { if err != nil { return nil, fmt.Errorf("error creating TLS config: %w", err) } - httpClient.Transport = kedautil.CreateRTWithTLSConfig(tlsConfig) + httpClient.Transport = kedautil.CreateHTTPTransportWithTLSConfig(tlsConfig) } return &artemisScaler{ diff --git a/pkg/scalers/aws/aws_sigv4.go b/pkg/scalers/aws/aws_sigv4.go index 6f5eb500c01..a3a8e45bcd1 100644 --- a/pkg/scalers/aws/aws_sigv4.go +++ b/pkg/scalers/aws/aws_sigv4.go @@ -101,7 +101,7 @@ func NewSigV4RoundTripper(config *scalersconfig.ScalerConfig, awsRegion string) client := amp.NewFromConfig(*awsCfg, func(_ *amp.Options) {}) rt := &roundTripper{ client: client, - next: httputils.CreateRT(false), + next: httputils.CreateHTTPTransport(false), } return rt, nil diff --git a/pkg/scalers/aws/aws_sigv4_test.go b/pkg/scalers/aws/aws_sigv4_test.go index 9c49be40143..8abc170aea0 100644 --- a/pkg/scalers/aws/aws_sigv4_test.go +++ b/pkg/scalers/aws/aws_sigv4_test.go @@ -36,7 +36,7 @@ import ( ) func TestSigV4RoundTripper(t *testing.T) { - transport := util.CreateRT(false) + transport := util.CreateHTTPTransport(false) cli := &http.Client{Transport: transport} diff --git a/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go index 123d06f47c2..a40d2b2deaa 100644 --- a/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go +++ b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go @@ -55,7 +55,7 @@ func TryAndGetAzureManagedPrometheusHTTPRoundTripper(logger logr.Logger, podIden return nil, err } - transport := util.CreateRT(false) + transport := util.CreateHTTPTransport(false) rt := &azureManagedPrometheusHTTPRoundTripper{ next: transport, chainedCredential: chainedCred, diff --git a/pkg/scalers/elasticsearch_scaler.go b/pkg/scalers/elasticsearch_scaler.go index 2b4aeea876c..dae001cea5c 100644 --- a/pkg/scalers/elasticsearch_scaler.go +++ b/pkg/scalers/elasticsearch_scaler.go @@ -130,7 +130,7 @@ func newElasticsearchClient(meta elasticsearchMetadata, logger logr.Logger) (*el } } - config.Transport = util.CreateRT(meta.UnsafeSsl) + config.Transport = util.CreateHTTPTransport(meta.UnsafeSsl) esClient, err := elasticsearch.NewClient(config) if err != nil { logger.Error(err, fmt.Sprintf("Found error when creating client: %s", err)) diff --git a/pkg/scalers/github_runner_scaler.go b/pkg/scalers/github_runner_scaler.go index 5086d44e157..448c687ec94 100644 --- a/pkg/scalers/github_runner_scaler.go +++ b/pkg/scalers/github_runner_scaler.go @@ -347,7 +347,7 @@ func NewGitHubRunnerScaler(config *scalersconfig.ScalerConfig) (Scaler, error) { } if meta.ApplicationID != 0 && meta.InstallationID != 0 && meta.ApplicationKey != "" { - httpTrans := kedautil.CreateRT(false) + httpTrans := kedautil.CreateHTTPTransport(false) hc, err := gha.New(httpTrans, meta.ApplicationID, meta.InstallationID, []byte(meta.ApplicationKey)) if err != nil { return nil, fmt.Errorf("error creating GitHub App client: %w, \n appID: %d, instID: %d", err, meta.ApplicationID, meta.InstallationID) diff --git a/pkg/scalers/ibmmq_scaler.go b/pkg/scalers/ibmmq_scaler.go index 9fc032c9e5b..9eff4f96bbc 100644 --- a/pkg/scalers/ibmmq_scaler.go +++ b/pkg/scalers/ibmmq_scaler.go @@ -98,7 +98,7 @@ func NewIBMMQScaler(config *scalersconfig.ScalerConfig) (Scaler, error) { if err != nil { return nil, err } - httpClient.Transport = kedautil.CreateRTWithTLSConfig(tlsConfig) + httpClient.Transport = kedautil.CreateHTTPTransportWithTLSConfig(tlsConfig) } scaler := &ibmmqScaler{ diff --git a/pkg/scalers/metrics_api_scaler.go b/pkg/scalers/metrics_api_scaler.go index 1e47713aa7f..56eb802520c 100644 --- a/pkg/scalers/metrics_api_scaler.go +++ b/pkg/scalers/metrics_api_scaler.go @@ -101,7 +101,7 @@ func NewMetricsAPIScaler(config *scalersconfig.ScalerConfig, kubeClient client.C if err != nil { return nil, err } - httpClient.Transport = kedautil.CreateRTWithTLSConfig(tlsConfig) + httpClient.Transport = kedautil.CreateHTTPTransportWithTLSConfig(tlsConfig) } return &metricsAPIScaler{ diff --git a/pkg/scalers/pulsar_scaler.go b/pkg/scalers/pulsar_scaler.go index 4062e312c77..6b8d587cd8e 100644 --- a/pkg/scalers/pulsar_scaler.go +++ b/pkg/scalers/pulsar_scaler.go @@ -165,7 +165,7 @@ func NewPulsarScaler(config *scalersconfig.ScalerConfig) (Scaler, error) { if err != nil { return nil, err } - client.Transport = kedautil.CreateRTWithTLSConfig(tlsConfig) + client.Transport = kedautil.CreateHTTPTransportWithTLSConfig(tlsConfig) } if pulsarMetadata.PulsarAuth.EnabledBearerAuth() || pulsarMetadata.PulsarAuth.EnabledBasicAuth() { diff --git a/pkg/scalers/rabbitmq_scaler.go b/pkg/scalers/rabbitmq_scaler.go index 04d666750dd..643549f2c65 100644 --- a/pkg/scalers/rabbitmq_scaler.go +++ b/pkg/scalers/rabbitmq_scaler.go @@ -298,7 +298,7 @@ func NewRabbitMQScaler(config *scalersconfig.ScalerConfig) (Scaler, error) { if tlsErr != nil { return nil, tlsErr } - s.httpClient.Transport = kedautil.CreateRTWithTLSConfig(tlsConfig) + s.httpClient.Transport = kedautil.CreateHTTPTransportWithTLSConfig(tlsConfig) } if meta.Protocol == amqpProtocol { diff --git a/pkg/util/http.go b/pkg/util/http.go index 090d0b26266..46249c9628d 100644 --- a/pkg/util/http.go +++ b/pkg/util/http.go @@ -50,7 +50,7 @@ func CreateHTTPClient(timeout time.Duration, unsafeSsl bool) *http.Client { if timeout <= 0 { timeout = 300 * time.Millisecond } - transport := CreateRT(unsafeSsl) + transport := CreateHTTPTransport(unsafeSsl) httpClient := &http.Client{ Timeout: timeout, Transport: transport, @@ -58,15 +58,15 @@ func CreateHTTPClient(timeout time.Duration, unsafeSsl bool) *http.Client { return httpClient } -// CreateRT returns a new instrumented HTTP RoundTripper with Proxy and Keep-alive settings. +// CreateHTTPTransport returns a new instrumented HTTP RoundTripper with Proxy and Keep-alive settings. // unsafeSsl parameter allows to avoid tls cert validation if it's required. -func CreateRT(unsafeSsl bool) http.RoundTripper { - return CreateRTWithTLSConfig(CreateTLSClientConfig(unsafeSsl)) +func CreateHTTPTransport(unsafeSsl bool) http.RoundTripper { + return CreateHTTPTransportWithTLSConfig(CreateTLSClientConfig(unsafeSsl)) } -// CreateRTWithTLSConfig returns a new instrumented HTTP RoundTripper with Proxy and Keep-alive -// settings using the given tls.Config. -func CreateRTWithTLSConfig(config *tls.Config) http.RoundTripper { +// CreateHTTPTransportWithTLSConfig returns a new instrumented HTTP RoundTripper with Proxy and +// Keep-alive settings using the given tls.Config. +func CreateHTTPTransportWithTLSConfig(config *tls.Config) http.RoundTripper { transport := &http.Transport{ TLSClientConfig: config, Proxy: http.ProxyFromEnvironment, diff --git a/pkg/util/http_roundtripper_test.go b/pkg/util/http_roundtripper_test.go index 2fde011feff..66f613af775 100644 --- a/pkg/util/http_roundtripper_test.go +++ b/pkg/util/http_roundtripper_test.go @@ -127,8 +127,8 @@ func TestCreateHTTPClient_TransportIsInstrumented(t *testing.T) { assert.True(t, ok, "expected CreateHTTPClient to wrap transport with InstrumentedRoundTripper") } -func TestCreateRTWithTLSConfig_IsInstrumented(t *testing.T) { - rt := CreateRTWithTLSConfig(nil) +func TestCreateHTTPTransportWithTLSConfig_IsInstrumented(t *testing.T) { + rt := CreateHTTPTransportWithTLSConfig(nil) _, ok := rt.(*InstrumentedRoundTripper) - assert.True(t, ok, "expected CreateRTWithTLSConfig to return an InstrumentedRoundTripper") + assert.True(t, ok, "expected CreateHTTPTransportWithTLSConfig to return an InstrumentedRoundTripper") } From f131cc12115203455f56832cf119f5f6707106d8 Mon Sep 17 00:00:00 2001 From: Ali Aqel Date: Wed, 22 Apr 2026 04:15:07 +0000 Subject: [PATCH 16/22] refactor: rename CreateHTTPTransport* to CreateRT* to reflect RoundTripper return type CreateHTTPTransport and CreateHTTPTransportWithTLSConfig now return http.RoundTripper (wrapping InstrumentedRoundTripper), so the *Transport suffix is misleading. Rename to CreateRT / CreateRTWithTLSConfig. All callers assign to http.Client.Transport or struct fields typed as http.RoundTripper, so the return type change is non-breaking. Signed-off-by: Ali Aqel --- pkg/scalers/artemis_scaler.go | 2 +- pkg/scalers/aws/aws_sigv4.go | 2 +- pkg/scalers/aws/aws_sigv4_test.go | 2 +- .../azure_managed_prometheus_http_round_tripper.go | 2 +- pkg/scalers/elasticsearch_scaler.go | 2 +- pkg/scalers/github_runner_scaler.go | 2 +- pkg/scalers/ibmmq_scaler.go | 2 +- pkg/scalers/metrics_api_scaler.go | 2 +- pkg/scalers/pulsar_scaler.go | 2 +- pkg/scalers/rabbitmq_scaler.go | 2 +- pkg/util/http.go | 12 ++++++------ pkg/util/http_roundtripper_test.go | 6 +++--- 12 files changed, 19 insertions(+), 19 deletions(-) diff --git a/pkg/scalers/artemis_scaler.go b/pkg/scalers/artemis_scaler.go index f031f54efac..89d425b0c87 100644 --- a/pkg/scalers/artemis_scaler.go +++ b/pkg/scalers/artemis_scaler.go @@ -109,7 +109,7 @@ func NewArtemisQueueScaler(config *scalersconfig.ScalerConfig) (Scaler, error) { if err != nil { return nil, fmt.Errorf("error creating TLS config: %w", err) } - httpClient.Transport = kedautil.CreateHTTPTransportWithTLSConfig(tlsConfig) + httpClient.Transport = kedautil.CreateRTWithTLSConfig(tlsConfig) } return &artemisScaler{ diff --git a/pkg/scalers/aws/aws_sigv4.go b/pkg/scalers/aws/aws_sigv4.go index a3a8e45bcd1..6f5eb500c01 100644 --- a/pkg/scalers/aws/aws_sigv4.go +++ b/pkg/scalers/aws/aws_sigv4.go @@ -101,7 +101,7 @@ func NewSigV4RoundTripper(config *scalersconfig.ScalerConfig, awsRegion string) client := amp.NewFromConfig(*awsCfg, func(_ *amp.Options) {}) rt := &roundTripper{ client: client, - next: httputils.CreateHTTPTransport(false), + next: httputils.CreateRT(false), } return rt, nil diff --git a/pkg/scalers/aws/aws_sigv4_test.go b/pkg/scalers/aws/aws_sigv4_test.go index 8abc170aea0..9c49be40143 100644 --- a/pkg/scalers/aws/aws_sigv4_test.go +++ b/pkg/scalers/aws/aws_sigv4_test.go @@ -36,7 +36,7 @@ import ( ) func TestSigV4RoundTripper(t *testing.T) { - transport := util.CreateHTTPTransport(false) + transport := util.CreateRT(false) cli := &http.Client{Transport: transport} diff --git a/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go index a40d2b2deaa..123d06f47c2 100644 --- a/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go +++ b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go @@ -55,7 +55,7 @@ func TryAndGetAzureManagedPrometheusHTTPRoundTripper(logger logr.Logger, podIden return nil, err } - transport := util.CreateHTTPTransport(false) + transport := util.CreateRT(false) rt := &azureManagedPrometheusHTTPRoundTripper{ next: transport, chainedCredential: chainedCred, diff --git a/pkg/scalers/elasticsearch_scaler.go b/pkg/scalers/elasticsearch_scaler.go index dae001cea5c..2b4aeea876c 100644 --- a/pkg/scalers/elasticsearch_scaler.go +++ b/pkg/scalers/elasticsearch_scaler.go @@ -130,7 +130,7 @@ func newElasticsearchClient(meta elasticsearchMetadata, logger logr.Logger) (*el } } - config.Transport = util.CreateHTTPTransport(meta.UnsafeSsl) + config.Transport = util.CreateRT(meta.UnsafeSsl) esClient, err := elasticsearch.NewClient(config) if err != nil { logger.Error(err, fmt.Sprintf("Found error when creating client: %s", err)) diff --git a/pkg/scalers/github_runner_scaler.go b/pkg/scalers/github_runner_scaler.go index 448c687ec94..5086d44e157 100644 --- a/pkg/scalers/github_runner_scaler.go +++ b/pkg/scalers/github_runner_scaler.go @@ -347,7 +347,7 @@ func NewGitHubRunnerScaler(config *scalersconfig.ScalerConfig) (Scaler, error) { } if meta.ApplicationID != 0 && meta.InstallationID != 0 && meta.ApplicationKey != "" { - httpTrans := kedautil.CreateHTTPTransport(false) + httpTrans := kedautil.CreateRT(false) hc, err := gha.New(httpTrans, meta.ApplicationID, meta.InstallationID, []byte(meta.ApplicationKey)) if err != nil { return nil, fmt.Errorf("error creating GitHub App client: %w, \n appID: %d, instID: %d", err, meta.ApplicationID, meta.InstallationID) diff --git a/pkg/scalers/ibmmq_scaler.go b/pkg/scalers/ibmmq_scaler.go index 9eff4f96bbc..9fc032c9e5b 100644 --- a/pkg/scalers/ibmmq_scaler.go +++ b/pkg/scalers/ibmmq_scaler.go @@ -98,7 +98,7 @@ func NewIBMMQScaler(config *scalersconfig.ScalerConfig) (Scaler, error) { if err != nil { return nil, err } - httpClient.Transport = kedautil.CreateHTTPTransportWithTLSConfig(tlsConfig) + httpClient.Transport = kedautil.CreateRTWithTLSConfig(tlsConfig) } scaler := &ibmmqScaler{ diff --git a/pkg/scalers/metrics_api_scaler.go b/pkg/scalers/metrics_api_scaler.go index 56eb802520c..1e47713aa7f 100644 --- a/pkg/scalers/metrics_api_scaler.go +++ b/pkg/scalers/metrics_api_scaler.go @@ -101,7 +101,7 @@ func NewMetricsAPIScaler(config *scalersconfig.ScalerConfig, kubeClient client.C if err != nil { return nil, err } - httpClient.Transport = kedautil.CreateHTTPTransportWithTLSConfig(tlsConfig) + httpClient.Transport = kedautil.CreateRTWithTLSConfig(tlsConfig) } return &metricsAPIScaler{ diff --git a/pkg/scalers/pulsar_scaler.go b/pkg/scalers/pulsar_scaler.go index 6b8d587cd8e..4062e312c77 100644 --- a/pkg/scalers/pulsar_scaler.go +++ b/pkg/scalers/pulsar_scaler.go @@ -165,7 +165,7 @@ func NewPulsarScaler(config *scalersconfig.ScalerConfig) (Scaler, error) { if err != nil { return nil, err } - client.Transport = kedautil.CreateHTTPTransportWithTLSConfig(tlsConfig) + client.Transport = kedautil.CreateRTWithTLSConfig(tlsConfig) } if pulsarMetadata.PulsarAuth.EnabledBearerAuth() || pulsarMetadata.PulsarAuth.EnabledBasicAuth() { diff --git a/pkg/scalers/rabbitmq_scaler.go b/pkg/scalers/rabbitmq_scaler.go index 643549f2c65..04d666750dd 100644 --- a/pkg/scalers/rabbitmq_scaler.go +++ b/pkg/scalers/rabbitmq_scaler.go @@ -298,7 +298,7 @@ func NewRabbitMQScaler(config *scalersconfig.ScalerConfig) (Scaler, error) { if tlsErr != nil { return nil, tlsErr } - s.httpClient.Transport = kedautil.CreateHTTPTransportWithTLSConfig(tlsConfig) + s.httpClient.Transport = kedautil.CreateRTWithTLSConfig(tlsConfig) } if meta.Protocol == amqpProtocol { diff --git a/pkg/util/http.go b/pkg/util/http.go index 46249c9628d..823105a8255 100644 --- a/pkg/util/http.go +++ b/pkg/util/http.go @@ -50,7 +50,7 @@ func CreateHTTPClient(timeout time.Duration, unsafeSsl bool) *http.Client { if timeout <= 0 { timeout = 300 * time.Millisecond } - transport := CreateHTTPTransport(unsafeSsl) + transport := CreateRT(unsafeSsl) httpClient := &http.Client{ Timeout: timeout, Transport: transport, @@ -58,15 +58,15 @@ func CreateHTTPClient(timeout time.Duration, unsafeSsl bool) *http.Client { return httpClient } -// CreateHTTPTransport returns a new instrumented HTTP RoundTripper with Proxy and Keep-alive settings. +// CreateRT returns a new instrumented HTTP RoundTripper with Proxy and Keep-alive settings. // unsafeSsl parameter allows to avoid tls cert validation if it's required. -func CreateHTTPTransport(unsafeSsl bool) http.RoundTripper { - return CreateHTTPTransportWithTLSConfig(CreateTLSClientConfig(unsafeSsl)) +func CreateRT(unsafeSsl bool) http.RoundTripper { + return CreateRTWithTLSConfig(CreateTLSClientConfig(unsafeSsl)) } -// CreateHTTPTransportWithTLSConfig returns a new instrumented HTTP RoundTripper with Proxy and +// CreateRTWithTLSConfig returns a new instrumented HTTP RoundTripper with Proxy and // Keep-alive settings using the given tls.Config. -func CreateHTTPTransportWithTLSConfig(config *tls.Config) http.RoundTripper { +func CreateRTWithTLSConfig(config *tls.Config) http.RoundTripper { transport := &http.Transport{ TLSClientConfig: config, Proxy: http.ProxyFromEnvironment, diff --git a/pkg/util/http_roundtripper_test.go b/pkg/util/http_roundtripper_test.go index 66f613af775..2fde011feff 100644 --- a/pkg/util/http_roundtripper_test.go +++ b/pkg/util/http_roundtripper_test.go @@ -127,8 +127,8 @@ func TestCreateHTTPClient_TransportIsInstrumented(t *testing.T) { assert.True(t, ok, "expected CreateHTTPClient to wrap transport with InstrumentedRoundTripper") } -func TestCreateHTTPTransportWithTLSConfig_IsInstrumented(t *testing.T) { - rt := CreateHTTPTransportWithTLSConfig(nil) +func TestCreateRTWithTLSConfig_IsInstrumented(t *testing.T) { + rt := CreateRTWithTLSConfig(nil) _, ok := rt.(*InstrumentedRoundTripper) - assert.True(t, ok, "expected CreateHTTPTransportWithTLSConfig to return an InstrumentedRoundTripper") + assert.True(t, ok, "expected CreateRTWithTLSConfig to return an InstrumentedRoundTripper") } From ba9aada48c7132900e1f125a8285a0b905a90005 Mon Sep 17 00:00:00 2001 From: Ali Aqel Date: Wed, 22 Apr 2026 04:17:51 +0000 Subject: [PATCH 17/22] docs: update RecordHTTPClientRequest comment per Copilot suggestion Signed-off-by: Ali Aqel --- pkg/metricscollector/metricscollectors.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/metricscollector/metricscollectors.go b/pkg/metricscollector/metricscollectors.go index c3f62f768fc..7c77d6dca96 100644 --- a/pkg/metricscollector/metricscollectors.go +++ b/pkg/metricscollector/metricscollectors.go @@ -82,9 +82,9 @@ type MetricsCollector interface { RecordCloudEventQueueStatus(namespace string, value int) // RecordHTTPClientRequest records the duration and outcome of an outbound HTTP request - // made by one of KEDA's internal HTTP clients. The scaler, triggerName, metricName, - // namespace, and scaledResource values are extracted from context keys by - // InstrumentedRoundTripper in the util package before this method is called. + // made by one of KEDA's internal HTTP clients. scaler, triggerName, metricName, + // namespace, and scaledResource are provided explicitly by the caller; upstream + // instrumentation may derive them from context before invoking the collector. RecordHTTPClientRequest(durationSeconds float64, statusCode int, isError bool, scaler, triggerName, metricName, namespace, scaledResource string) } From c0efe49ee2eb001eaae9ce254330e8976203a5f4 Mon Sep 17 00:00:00 2001 From: Ali Aqel Date: Thu, 23 Apr 2026 04:58:22 +0000 Subject: [PATCH 18/22] refactor: move HTTP client metrics helpers into metricscollector Signed-off-by: Ali Aqel --- .../http_roundtripper.go | 10 +- .../http_roundtripper_test.go | 193 ++++++++++++++++++ pkg/metricscollector/request_context.go | 34 +++ pkg/scaling/cache/scalers_cache.go | 15 +- pkg/scaling/cache/scalers_cache_test.go | 14 +- pkg/util/http.go | 4 +- .../http_roundtripper_integration_test.go | 31 +++ pkg/util/http_roundtripper_test.go | 134 ------------ .../opentelemetry_metrics_test.go | 46 ++++- .../prometheus_metrics_test.go | 46 ++++- 10 files changed, 351 insertions(+), 176 deletions(-) rename pkg/{util => metricscollector}/http_roundtripper.go (89%) create mode 100644 pkg/metricscollector/http_roundtripper_test.go create mode 100644 pkg/metricscollector/request_context.go create mode 100644 pkg/util/http_roundtripper_integration_test.go delete mode 100644 pkg/util/http_roundtripper_test.go diff --git a/pkg/util/http_roundtripper.go b/pkg/metricscollector/http_roundtripper.go similarity index 89% rename from pkg/util/http_roundtripper.go rename to pkg/metricscollector/http_roundtripper.go index 4af4d4a283d..289e4a758f3 100644 --- a/pkg/util/http_roundtripper.go +++ b/pkg/metricscollector/http_roundtripper.go @@ -14,13 +14,11 @@ See the License for the specific language governing permissions and limitations under the License. */ -package util +package metricscollector import ( "net/http" "time" - - "github.com/kedacore/keda/v2/pkg/metricscollector" ) // contextKey is an unexported type for context keys defined in this package, @@ -81,16 +79,16 @@ func (r *InstrumentedRoundTripper) RoundTrip(req *http.Request) (*http.Response, scaledResource, srOK := ctx.Value(ScaledResourceContextKey).(string) // Only record metrics for scaler metric collection requests, identified by the - // presence of all five context keys injected by buildScalerRequestCtx. + // presence of all five context keys injected by BuildScalerRequestCtx. // Other HTTP calls (e.g. during scaler initialization) are not recorded. if !scalerOK || !triggerOK || !metricOK || !nsOK || !srOK { return resp, err } if err != nil { - metricscollector.RecordHTTPClientRequest(duration, 0, true, scaler, triggerName, metricName, namespace, scaledResource) + RecordHTTPClientRequest(duration, 0, true, scaler, triggerName, metricName, namespace, scaledResource) return nil, err } - metricscollector.RecordHTTPClientRequest(duration, resp.StatusCode, false, scaler, triggerName, metricName, namespace, scaledResource) + RecordHTTPClientRequest(duration, resp.StatusCode, false, scaler, triggerName, metricName, namespace, scaledResource) return resp, nil } diff --git a/pkg/metricscollector/http_roundtripper_test.go b/pkg/metricscollector/http_roundtripper_test.go new file mode 100644 index 00000000000..8d49da20205 --- /dev/null +++ b/pkg/metricscollector/http_roundtripper_test.go @@ -0,0 +1,193 @@ +/* +Copyright 2026 The KEDA Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metricscollector + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" + "testing" + + . "github.com/onsi/gomega" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" +) + +type mockRoundTripper struct { + resp *http.Response + err error +} + +func (m *mockRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + if m.resp != nil && m.resp.Request == nil { + m.resp.Request = req + } + + return m.resp, m.err +} + +func fakeResponse(statusCode int) *http.Response { + return &http.Response{ + StatusCode: statusCode, + Body: io.NopCloser(strings.NewReader("")), + } +} + +func withPromCollector(t *testing.T) { + t.Helper() + + previousCollectors := collectors + collectors = []MetricsCollector{&PromMetrics{}} + t.Cleanup(func() { + collectors = previousCollectors + }) +} + +func TestInstrumentedRoundTripper_RecordsSuccessfulResponses(t *testing.T) { + testCases := []struct { + name string + statusCode int + }{ + {name: "2xx response", statusCode: http.StatusOK}, + {name: "4xx response", statusCode: http.StatusBadRequest}, + {name: "5xx response", statusCode: http.StatusInternalServerError}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + RegisterTestingT(t) + withPromCollector(t) + + scalerName := fmt.Sprintf("prometheus-%d", tc.statusCode) + triggerName := fmt.Sprintf("trigger-%d", tc.statusCode) + metricName := fmt.Sprintf("metric-%d", tc.statusCode) + scaledResource := fmt.Sprintf("so-%d", tc.statusCode) + + rt := NewInstrumentedRoundTripper(&mockRoundTripper{resp: fakeResponse(tc.statusCode)}) + + ctx := context.Background() + ctx = context.WithValue(ctx, ScalerContextKey, scalerName) + ctx = context.WithValue(ctx, TriggerNameContextKey, triggerName) + ctx = context.WithValue(ctx, MetricNameContextKey, metricName) + ctx = context.WithValue(ctx, NamespaceContextKey, "default") + ctx = context.WithValue(ctx, ScaledResourceContextKey, scaledResource) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://example.com", nil) + Expect(err).To(BeNil()) + + resp, err := rt.RoundTrip(req) + Expect(err).To(BeNil()) + Expect(resp).NotTo(BeNil()) + + counterValue, err := httpClientRequestsTotal. + GetMetricWithLabelValues("default", scaledResource, scalerName, triggerName, metricName, fmt.Sprintf("%d", tc.statusCode)) + Expect(err).To(BeNil()) + Expect(counterValue).NotTo(BeNil()) + m := &dto.Metric{} + err = counterValue.Write(m) + Expect(err).To(BeNil()) + Expect(m.Counter.GetValue()).To(BeNumerically("==", 1)) + + durationHistogram, err := httpClientRequestDuration. + GetMetricWithLabelValues(scalerName, fmt.Sprintf("%d", tc.statusCode)) + Expect(err).To(BeNil()) + Expect(durationHistogram).NotTo(BeNil()) + err = durationHistogram.(prometheus.Metric).Write(m) + Expect(err).To(BeNil()) + Expect(m.Histogram.GetSampleCount()).To(BeNumerically("==", 1)) + }) + } +} + +func TestInstrumentedRoundTripper_RecordsTransportErrors(t *testing.T) { + RegisterTestingT(t) + withPromCollector(t) + + scalerName := "prometheus-transport-error" + triggerName := "trigger-transport-error" + metricName := "metric-transport-error" + scaledResource := "so-transport-error" + + transportErr := io.ErrUnexpectedEOF + rt := NewInstrumentedRoundTripper(&mockRoundTripper{err: transportErr}) + + ctx := context.Background() + ctx = context.WithValue(ctx, ScalerContextKey, scalerName) + ctx = context.WithValue(ctx, TriggerNameContextKey, triggerName) + ctx = context.WithValue(ctx, MetricNameContextKey, metricName) + ctx = context.WithValue(ctx, NamespaceContextKey, "default") + ctx = context.WithValue(ctx, ScaledResourceContextKey, scaledResource) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://example.com", nil) + Expect(err).To(BeNil()) + + resp, err := rt.RoundTrip(req) + Expect(err).To(Equal(transportErr)) + Expect(resp).To(BeNil()) + + counterValue, err := httpClientRequestsTotal. + GetMetricWithLabelValues("default", scaledResource, scalerName, triggerName, metricName, "error") + Expect(err).To(BeNil()) + Expect(counterValue).NotTo(BeNil()) + m := &dto.Metric{} + err = counterValue.Write(m) + Expect(err).To(BeNil()) + Expect(m.Counter.GetValue()).To(BeNumerically("==", 1)) + + durationHistogram, err := httpClientRequestDuration. + GetMetricWithLabelValues(scalerName, "error") + Expect(err).To(BeNil()) + Expect(durationHistogram).NotTo(BeNil()) + err = durationHistogram.(prometheus.Metric).Write(m) + Expect(err).To(BeNil()) + Expect(m.Histogram.GetSampleCount()).To(BeNumerically("==", 1)) +} + +func TestInstrumentedRoundTripper_ResponseReturnedUnmodified(t *testing.T) { + RegisterTestingT(t) + + expected := fakeResponse(http.StatusAccepted) + rt := NewInstrumentedRoundTripper(&mockRoundTripper{resp: expected}) + + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, "http://example.com", nil) + Expect(err).To(BeNil()) + + got, err := rt.RoundTrip(req) + Expect(err).To(BeNil()) + Expect(got).To(Equal(expected)) +} + +func TestInstrumentedRoundTripper_NilNextUsesDefault(t *testing.T) { + RegisterTestingT(t) + + rt := NewInstrumentedRoundTripper(nil) + Expect(fmt.Sprintf("%T", rt)).To(Equal("*metricscollector.InstrumentedRoundTripper")) +} + +func TestInstrumentedRoundTripper_ScalerContextKey_Missing(t *testing.T) { + RegisterTestingT(t) + rt := NewInstrumentedRoundTripper(&mockRoundTripper{resp: fakeResponse(200)}) + + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, "http://example.com", nil) + Expect(err).To(BeNil()) + + resp, err := rt.RoundTrip(req) + Expect(err).To(BeNil()) + Expect(resp).NotTo(BeNil()) +} diff --git a/pkg/metricscollector/request_context.go b/pkg/metricscollector/request_context.go new file mode 100644 index 00000000000..d775c91766f --- /dev/null +++ b/pkg/metricscollector/request_context.go @@ -0,0 +1,34 @@ +/* +Copyright 2026 The KEDA Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metricscollector + +import ( + "context" + + "github.com/kedacore/keda/v2/pkg/scalers/scalersconfig" +) + +// BuildScalerRequestCtx attaches scaler metadata used by HTTP client +// instrumentation to the outbound request context. +func BuildScalerRequestCtx(ctx context.Context, config scalersconfig.ScalerConfig, metricName string) context.Context { + requestCtx := context.WithValue(ctx, ScalerContextKey, config.TriggerType) + requestCtx = context.WithValue(requestCtx, TriggerNameContextKey, config.TriggerName) + requestCtx = context.WithValue(requestCtx, MetricNameContextKey, metricName) + requestCtx = context.WithValue(requestCtx, NamespaceContextKey, config.ScalableObjectNamespace) + requestCtx = context.WithValue(requestCtx, ScaledResourceContextKey, config.ScalableObjectName) + return requestCtx +} diff --git a/pkg/scaling/cache/scalers_cache.go b/pkg/scaling/cache/scalers_cache.go index 065676f2312..3e93fa7c165 100644 --- a/pkg/scaling/cache/scalers_cache.go +++ b/pkg/scaling/cache/scalers_cache.go @@ -29,9 +29,9 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" + "github.com/kedacore/keda/v2/pkg/metricscollector" "github.com/kedacore/keda/v2/pkg/scalers" "github.com/kedacore/keda/v2/pkg/scalers/scalersconfig" - kedautil "github.com/kedacore/keda/v2/pkg/util" ) var log = logf.Log.WithName("scalers_cache") @@ -150,15 +150,6 @@ func (c *ScalersCache) GetMetricSpecForScalingForScaler(ctx context.Context, ind return metricSpecs, err } -func buildScalerRequestCtx(ctx context.Context, sb ScalerBuilder, metricName string) context.Context { - requestCtx := context.WithValue(ctx, kedautil.ScalerContextKey, sb.ScalerConfig.TriggerType) - requestCtx = context.WithValue(requestCtx, kedautil.TriggerNameContextKey, sb.ScalerConfig.TriggerName) - requestCtx = context.WithValue(requestCtx, kedautil.MetricNameContextKey, metricName) - requestCtx = context.WithValue(requestCtx, kedautil.NamespaceContextKey, sb.ScalerConfig.ScalableObjectNamespace) - requestCtx = context.WithValue(requestCtx, kedautil.ScaledResourceContextKey, sb.ScalerConfig.ScalableObjectName) - return requestCtx -} - // GetMetricsAndActivityForScaler returns metric value, activity and latency for a scaler identified by the metric name // and by the input index (from the list of scalers in this ScaledObject) func (c *ScalersCache) GetMetricsAndActivityForScaler(ctx context.Context, index int, metricName string) ([]external_metrics.ExternalMetricValue, bool, time.Duration, error) { @@ -166,7 +157,7 @@ func (c *ScalersCache) GetMetricsAndActivityForScaler(ctx context.Context, index if err != nil { return nil, false, -1, err } - requestCtx := buildScalerRequestCtx(ctx, sb, metricName) + requestCtx := metricscollector.BuildScalerRequestCtx(ctx, sb.ScalerConfig, metricName) startTime := time.Now() metric, activity, err := sb.Scaler.GetMetricsAndActivity(requestCtx, metricName) if err == nil { @@ -181,7 +172,7 @@ func (c *ScalersCache) GetMetricsAndActivityForScaler(ctx context.Context, index if err != nil { return nil, false, -1, err } - requestCtx = buildScalerRequestCtx(ctx, newSb, metricName) + requestCtx = metricscollector.BuildScalerRequestCtx(ctx, newSb.ScalerConfig, metricName) startTime = time.Now() metric, activity, err = ns.GetMetricsAndActivity(requestCtx, metricName) return metric, activity, time.Since(startTime), err diff --git a/pkg/scaling/cache/scalers_cache_test.go b/pkg/scaling/cache/scalers_cache_test.go index 268484448bc..584af1ed5aa 100644 --- a/pkg/scaling/cache/scalers_cache_test.go +++ b/pkg/scaling/cache/scalers_cache_test.go @@ -22,8 +22,8 @@ import ( . "github.com/onsi/gomega" + "github.com/kedacore/keda/v2/pkg/metricscollector" "github.com/kedacore/keda/v2/pkg/scalers/scalersconfig" - kedautil "github.com/kedacore/keda/v2/pkg/util" ) func TestBuildScalerRequestCtx(t *testing.T) { @@ -38,13 +38,13 @@ func TestBuildScalerRequestCtx(t *testing.T) { }, } - ctx := buildScalerRequestCtx(context.Background(), sb, "my-metric") + ctx := metricscollector.BuildScalerRequestCtx(context.Background(), sb.ScalerConfig, "my-metric") - Expect(ctx.Value(kedautil.ScalerContextKey)).To(Equal("prometheus")) - Expect(ctx.Value(kedautil.TriggerNameContextKey)).To(Equal("my-trigger")) - Expect(ctx.Value(kedautil.MetricNameContextKey)).To(Equal("my-metric")) - Expect(ctx.Value(kedautil.NamespaceContextKey)).To(Equal("my-namespace")) - Expect(ctx.Value(kedautil.ScaledResourceContextKey)).To(Equal("my-scaled-object")) + Expect(ctx.Value(metricscollector.ScalerContextKey)).To(Equal("prometheus")) + Expect(ctx.Value(metricscollector.TriggerNameContextKey)).To(Equal("my-trigger")) + Expect(ctx.Value(metricscollector.MetricNameContextKey)).To(Equal("my-metric")) + Expect(ctx.Value(metricscollector.NamespaceContextKey)).To(Equal("my-namespace")) + Expect(ctx.Value(metricscollector.ScaledResourceContextKey)).To(Equal("my-scaled-object")) } func TestEmptyScalersCache(t *testing.T) { diff --git a/pkg/util/http.go b/pkg/util/http.go index 823105a8255..538fb088754 100644 --- a/pkg/util/http.go +++ b/pkg/util/http.go @@ -20,6 +20,8 @@ import ( "crypto/tls" "net/http" "time" + + "github.com/kedacore/keda/v2/pkg/metricscollector" ) var disableKeepAlives bool @@ -76,5 +78,5 @@ func CreateRTWithTLSConfig(config *tls.Config) http.RoundTripper { transport.DisableKeepAlives = true transport.IdleConnTimeout = 100 * time.Second } - return NewInstrumentedRoundTripper(transport) + return metricscollector.NewInstrumentedRoundTripper(transport) } diff --git a/pkg/util/http_roundtripper_integration_test.go b/pkg/util/http_roundtripper_integration_test.go new file mode 100644 index 00000000000..fe1a30ab24e --- /dev/null +++ b/pkg/util/http_roundtripper_integration_test.go @@ -0,0 +1,31 @@ +/* +Copyright 2026 The KEDA Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "fmt" + "testing" + + . "github.com/onsi/gomega" +) + +func TestCreateHTTPClient_TransportIsInstrumented(t *testing.T) { + RegisterTestingT(t) + + client := CreateHTTPClient(0, false) + Expect(fmt.Sprintf("%T", client.Transport)).To(Equal("*metricscollector.InstrumentedRoundTripper")) +} diff --git a/pkg/util/http_roundtripper_test.go b/pkg/util/http_roundtripper_test.go deleted file mode 100644 index 2fde011feff..00000000000 --- a/pkg/util/http_roundtripper_test.go +++ /dev/null @@ -1,134 +0,0 @@ -/* -Copyright 2026 The KEDA Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "context" - "errors" - "io" - "net/http" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -type mockRoundTripper struct { - resp *http.Response - err error -} - -func (m *mockRoundTripper) RoundTrip(_ *http.Request) (*http.Response, error) { - return m.resp, m.err -} - -func fakeResponse(statusCode int) *http.Response { - return &http.Response{ - StatusCode: statusCode, - Body: io.NopCloser(strings.NewReader("")), - } -} - -func newRequest(ctx context.Context) *http.Request { - req, _ := http.NewRequestWithContext(ctx, http.MethodGet, "http://example.com", nil) - return req -} - -func TestInstrumentedRoundTripper_SuccessfulRequest(t *testing.T) { - for _, statusCode := range []int{200, 201, 301, 400, 500} { - t.Run(http.StatusText(statusCode), func(t *testing.T) { - rt := NewInstrumentedRoundTripper(&mockRoundTripper{resp: fakeResponse(statusCode)}) - - resp, err := rt.RoundTrip(newRequest(context.Background())) - - require.NoError(t, err) - defer resp.Body.Close() - assert.Equal(t, statusCode, resp.StatusCode) - }) - } -} - -func TestInstrumentedRoundTripper_TransportError(t *testing.T) { - transportErr := errors.New("connection refused") - rt := NewInstrumentedRoundTripper(&mockRoundTripper{err: transportErr}) - - resp, err := rt.RoundTrip(newRequest(context.Background())) //nolint:bodyclose // resp is nil on error - - assert.ErrorIs(t, err, transportErr) - assert.Nil(t, resp) -} - -func TestInstrumentedRoundTripper_ResponseReturnedUnmodified(t *testing.T) { - expected := fakeResponse(202) - rt := NewInstrumentedRoundTripper(&mockRoundTripper{resp: expected}) - - got, err := rt.RoundTrip(newRequest(context.Background())) - - require.NoError(t, err) - defer got.Body.Close() - assert.Same(t, expected, got) -} - -func TestInstrumentedRoundTripper_NilNextUsesDefault(t *testing.T) { - rt := NewInstrumentedRoundTripper(nil) - irt, ok := rt.(*InstrumentedRoundTripper) - require.True(t, ok) - assert.Equal(t, http.DefaultTransport, irt.next) -} - -func TestInstrumentedRoundTripper_ScalerContextKey_Missing(t *testing.T) { - // When one or more of the five required context keys are absent, the - // RoundTripper should not panic, complete normally, and skip metric recording. - rt := NewInstrumentedRoundTripper(&mockRoundTripper{resp: fakeResponse(200)}) - - resp, err := rt.RoundTrip(newRequest(context.Background())) - - require.NoError(t, err) - defer resp.Body.Close() - assert.Equal(t, 200, resp.StatusCode) -} - -func TestInstrumentedRoundTripper_AllContextKeys(t *testing.T) { - // When all five required context keys are present the RoundTripper should - // complete normally and forward the request to the underlying transport. - rt := NewInstrumentedRoundTripper(&mockRoundTripper{resp: fakeResponse(200)}) - - ctx := context.Background() - ctx = context.WithValue(ctx, ScalerContextKey, "prometheus") - ctx = context.WithValue(ctx, TriggerNameContextKey, "my-trigger") - ctx = context.WithValue(ctx, MetricNameContextKey, "my-metric") - ctx = context.WithValue(ctx, NamespaceContextKey, "default") - ctx = context.WithValue(ctx, ScaledResourceContextKey, "my-so") - resp, err := rt.RoundTrip(newRequest(ctx)) - - require.NoError(t, err) - defer resp.Body.Close() - assert.Equal(t, 200, resp.StatusCode) -} - -func TestCreateHTTPClient_TransportIsInstrumented(t *testing.T) { - client := CreateHTTPClient(0, false) - _, ok := client.Transport.(*InstrumentedRoundTripper) - assert.True(t, ok, "expected CreateHTTPClient to wrap transport with InstrumentedRoundTripper") -} - -func TestCreateRTWithTLSConfig_IsInstrumented(t *testing.T) { - rt := CreateRTWithTLSConfig(nil) - _, ok := rt.(*InstrumentedRoundTripper) - assert.True(t, ok, "expected CreateRTWithTLSConfig to return an InstrumentedRoundTripper") -} diff --git a/tests/sequential/opentelemetry_metrics/opentelemetry_metrics_test.go b/tests/sequential/opentelemetry_metrics/opentelemetry_metrics_test.go index 83bf7e6eb3e..007423a4adc 100644 --- a/tests/sequential/opentelemetry_metrics/opentelemetry_metrics_test.go +++ b/tests/sequential/opentelemetry_metrics/opentelemetry_metrics_test.go @@ -43,11 +43,13 @@ var ( deploymentName = fmt.Sprintf("%s-deployment", testName) monitoredDeploymentName = fmt.Sprintf("%s-monitored", testName) scaledObjectName = fmt.Sprintf("%s-so", testName) + httpClientScaledObjectName = fmt.Sprintf("%s-so-http-client", testName) wrongScaledObjectName = fmt.Sprintf("%s-so-wrong", testName) scaledObjectGrpcName = fmt.Sprintf("%s-so-grpc", testName) scaledJobName = fmt.Sprintf("%s-sj", testName) wrongScaledJobName = fmt.Sprintf("%s-sj-wrong", testName) wrongScalerName = fmt.Sprintf("%s-wrong-scaler", testName) + httpClientScalerName = fmt.Sprintf("%s-http-client-scaler", testName) cronScaledJobName = fmt.Sprintf("%s-cron-sj", testName) clientName = fmt.Sprintf("%s-client", testName) cloudEventSourceName = fmt.Sprintf("%s-ce", testName) @@ -69,11 +71,13 @@ type templateData struct { TestNamespace string DeploymentName string ScaledObjectName string + HTTPClientScaledObjectName string ScaledJobName string ScaledObjectGrpcName string WrongScaledObjectName string WrongScaledJobName string WrongScalerName string + HTTPClientScalerName string CronScaledJobName string MonitoredDeploymentName string ClientName string @@ -197,6 +201,30 @@ spec: query: 'keda_scaler_errors_total{namespace="{{.TestNamespace}}",scaledObject="{{.WrongScaledObjectName}}"}' ` + httpClientScaledObjectTemplate = ` +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: {{.HTTPClientScaledObjectName}} + namespace: {{.TestNamespace}} +spec: + scaleTargetRef: + name: {{.DeploymentName}} + pollingInterval: 2 + idleReplicaCount: 0 + minReplicaCount: 1 + maxReplicaCount: 2 + cooldownPeriod: 10 + triggers: + - type: prometheus + name: {{.HTTPClientScalerName}} + metadata: + serverAddress: http://keda-prometheus.keda.svc.cluster.local:8080 + metricName: keda_scaler_errors_total + threshold: '1' + query: 'keda_scaler_errors_total{namespace="{{.TestNamespace}}",scaledObject="{{.HTTPClientScaledObjectName}}"}' +` + scaledJobTemplate = ` apiVersion: keda.sh/v1alpha1 kind: ScaledJob @@ -502,11 +530,13 @@ func getTemplateData() (templateData, []Template) { TestNamespace: testNamespace, DeploymentName: deploymentName, ScaledObjectName: scaledObjectName, + HTTPClientScaledObjectName: httpClientScaledObjectName, WrongScaledObjectName: wrongScaledObjectName, ScaledObjectGrpcName: scaledObjectGrpcName, ScaledJobName: scaledJobName, WrongScaledJobName: wrongScaledJobName, WrongScalerName: wrongScalerName, + HTTPClientScalerName: httpClientScalerName, MonitoredDeploymentName: monitoredDeploymentName, ClientName: clientName, CronScaledJobName: cronScaledJobName, @@ -1268,13 +1298,13 @@ func testCloudEventEmittedError(t *testing.T, data templateData) { func testHTTPClientMetrics(t *testing.T, data templateData) { t.Log("--- testing HTTP client metrics ---") - // The wrongScaledObject uses a prometheus-type scaler that makes real HTTP - // requests on every poll interval, so its records should be present once at - // least one poll cycle has completed. + // The dedicated HTTP client metrics ScaledObject uses a prometheus-type + // scaler that makes real HTTP requests on every poll interval, so its + // records should be present once at least one poll cycle has completed. KubectlDeleteWithTemplate(t, data, "scaledObjectTemplate", scaledObjectTemplate) - KubectlApplyWithTemplate(t, data, "wrongScaledObjectTemplate", wrongScaledObjectTemplate) + KubectlApplyWithTemplate(t, data, "httpClientScaledObjectTemplate", httpClientScaledObjectTemplate) defer func() { - KubectlDeleteWithTemplate(t, data, "wrongScaledObjectTemplate", wrongScaledObjectTemplate) + KubectlDeleteWithTemplate(t, data, "httpClientScaledObjectTemplate", httpClientScaledObjectTemplate) KubectlApplyWithTemplate(t, data, "scaledObjectTemplate", scaledObjectTemplate) }() @@ -1284,9 +1314,9 @@ func testHTTPClientMetrics(t *testing.T, data templateData) { matchLabels := func(labels []*prommodel.LabelPair) bool { return ExtractPrometheusLabelValue("namespace", labels) == data.TestNamespace && - ExtractPrometheusLabelValue("scaled_resource", labels) == wrongScaledObjectName && + ExtractPrometheusLabelValue("scaled_resource", labels) == data.HTTPClientScaledObjectName && ExtractPrometheusLabelValue("scaler", labels) == "prometheus" && - ExtractPrometheusLabelValue("trigger_name", labels) == wrongScalerName && + ExtractPrometheusLabelValue("trigger_name", labels) == data.HTTPClientScalerName && ExtractPrometheusLabelValue("metric_name", labels) == "s0-prometheus" } @@ -1303,7 +1333,7 @@ func testHTTPClientMetrics(t *testing.T, data templateData) { } assert.True(t, found, "expected keda_scaler_http_requests_count_total with namespace=%s, scaled_resource=%s, scaler=prometheus, trigger_name=%s, metric_name=s0-prometheus", - data.TestNamespace, wrongScaledObjectName, wrongScalerName) + data.TestNamespace, data.HTTPClientScaledObjectName, data.HTTPClientScalerName) } matchHistogramLabels := func(labels []*prommodel.LabelPair) bool { diff --git a/tests/sequential/prometheus_metrics/prometheus_metrics_test.go b/tests/sequential/prometheus_metrics/prometheus_metrics_test.go index 212615b23b5..c6ff49a3dca 100644 --- a/tests/sequential/prometheus_metrics/prometheus_metrics_test.go +++ b/tests/sequential/prometheus_metrics/prometheus_metrics_test.go @@ -40,10 +40,12 @@ var ( deploymentName = fmt.Sprintf("%s-deployment", testName) monitoredDeploymentName = fmt.Sprintf("%s-monitored", testName) scaledObjectName = fmt.Sprintf("%s-so", testName) + httpClientScaledObjectName = fmt.Sprintf("%s-so-http-client", testName) wrongScaledObjectName = fmt.Sprintf("%s-so-wrong", testName) scaledJobName = fmt.Sprintf("%s-sj", testName) wrongScaledJobName = fmt.Sprintf("%s-sj-wrong", testName) wrongScalerName = fmt.Sprintf("%s-wrong-scaler", testName) + httpClientScalerName = fmt.Sprintf("%s-http-client-scaler", testName) cronScaledJobName = fmt.Sprintf("%s-cron-sj", testName) clientName = fmt.Sprintf("%s-client", testName) cloudEventSourceName = fmt.Sprintf("%s-ce", testName) @@ -62,10 +64,12 @@ type templateData struct { TestNamespace string DeploymentName string ScaledObjectName string + HTTPClientScaledObjectName string ScaledJobName string WrongScaledObjectName string WrongScaledJobName string WrongScalerName string + HTTPClientScalerName string CronScaledJobName string MonitoredDeploymentName string ClientName string @@ -168,6 +172,30 @@ spec: query: 'keda_scaler_errors_total{namespace="{{.TestNamespace}}",scaledObject="{{.WrongScaledObjectName}}"}' ` + httpClientScaledObjectTemplate = ` +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: {{.HTTPClientScaledObjectName}} + namespace: {{.TestNamespace}} +spec: + scaleTargetRef: + name: {{.DeploymentName}} + pollingInterval: 2 + idleReplicaCount: 0 + minReplicaCount: 1 + maxReplicaCount: 2 + cooldownPeriod: 10 + triggers: + - type: prometheus + name: {{.HTTPClientScalerName}} + metadata: + serverAddress: http://keda-prometheus.keda.svc.cluster.local:8080 + metricName: keda_scaler_errors_total + threshold: '1' + query: 'keda_scaler_errors_total{namespace="{{.TestNamespace}}",scaledObject="{{.HTTPClientScaledObjectName}}"}' +` + scaledJobTemplate = ` apiVersion: keda.sh/v1alpha1 kind: ScaledJob @@ -461,10 +489,12 @@ func getTemplateData() (templateData, []Template) { TestNamespace: testNamespace, DeploymentName: deploymentName, ScaledObjectName: scaledObjectName, + HTTPClientScaledObjectName: httpClientScaledObjectName, WrongScaledObjectName: wrongScaledObjectName, ScaledJobName: scaledJobName, WrongScaledJobName: wrongScaledJobName, WrongScalerName: wrongScalerName, + HTTPClientScalerName: httpClientScalerName, MonitoredDeploymentName: monitoredDeploymentName, ClientName: clientName, CronScaledJobName: cronScaledJobName, @@ -1435,21 +1465,21 @@ func testCloudEventEmittedError(t *testing.T, data templateData) { func testHTTPClientMetrics(t *testing.T, data templateData) { t.Log("--- testing HTTP client metrics ---") - // The wrongScaledObject uses a prometheus-type scaler that makes real HTTP - // requests on every poll interval, so its records should be present once at - // least one poll cycle has completed. + // The dedicated HTTP client metrics ScaledObject uses a prometheus-type + // scaler that makes real HTTP requests on every poll interval, so its + // records should be present once at least one poll cycle has completed. KubectlDeleteWithTemplate(t, data, "scaledObjectTemplate", scaledObjectTemplate) - KubectlApplyWithTemplate(t, data, "wrongScaledObjectTemplate", wrongScaledObjectTemplate) + KubectlApplyWithTemplate(t, data, "httpClientScaledObjectTemplate", httpClientScaledObjectTemplate) defer func() { - KubectlDeleteWithTemplate(t, data, "wrongScaledObjectTemplate", wrongScaledObjectTemplate) + KubectlDeleteWithTemplate(t, data, "httpClientScaledObjectTemplate", httpClientScaledObjectTemplate) KubectlApplyWithTemplate(t, data, "scaledObjectTemplate", scaledObjectTemplate) }() matchLabels := func(labels []*prommodel.LabelPair) bool { return ExtractPrometheusLabelValue("namespace", labels) == testNamespace && - ExtractPrometheusLabelValue("scaled_resource", labels) == wrongScaledObjectName && + ExtractPrometheusLabelValue("scaled_resource", labels) == data.HTTPClientScaledObjectName && ExtractPrometheusLabelValue("scaler", labels) == "prometheus" && - ExtractPrometheusLabelValue("trigger_name", labels) == wrongScalerName && + ExtractPrometheusLabelValue("trigger_name", labels) == data.HTTPClientScalerName && ExtractPrometheusLabelValue("metric_name", labels) == "s0-prometheus" } @@ -1465,7 +1495,7 @@ func testHTTPClientMetrics(t *testing.T, data templateData) { families := WaitForPrometheusMetric(t, "keda_scaler_http_requests_total", familyValidator) assert.True(t, familyValidator(families["keda_scaler_http_requests_total"]), "expected keda_scaler_http_requests_total with namespace=%s, scaled_resource=%s, scaler=prometheus, trigger_name=%s, metric_name=s0-prometheus", - testNamespace, wrongScaledObjectName, wrongScalerName) + testNamespace, data.HTTPClientScaledObjectName, data.HTTPClientScalerName) matchHistogramLabels := func(labels []*prommodel.LabelPair) bool { return ExtractPrometheusLabelValue("scaler", labels) == "prometheus" From 179604c128c5cc33cc088b43c95ea54e412adf97 Mon Sep 17 00:00:00 2001 From: Ali Aqel Date: Thu, 23 Apr 2026 05:09:55 +0000 Subject: [PATCH 19/22] refactor: colocate HTTP client metrics helpers Signed-off-by: Ali Aqel --- pkg/metricscollector/http_roundtripper.go | 14 ++++++++ pkg/metricscollector/request_context.go | 34 ------------------- pkg/scalers/aws/aws_sigv4_test.go | 4 +-- ...e_managed_prometheus_http_round_tripper.go | 4 +-- pkg/scalers/github_runner_scaler.go | 4 +-- .../http_roundtripper_integration_test.go | 31 ----------------- 6 files changed, 20 insertions(+), 71 deletions(-) delete mode 100644 pkg/metricscollector/request_context.go delete mode 100644 pkg/util/http_roundtripper_integration_test.go diff --git a/pkg/metricscollector/http_roundtripper.go b/pkg/metricscollector/http_roundtripper.go index 289e4a758f3..77554eeb8fe 100644 --- a/pkg/metricscollector/http_roundtripper.go +++ b/pkg/metricscollector/http_roundtripper.go @@ -17,8 +17,11 @@ limitations under the License. package metricscollector import ( + "context" "net/http" "time" + + "github.com/kedacore/keda/v2/pkg/scalers/scalersconfig" ) // contextKey is an unexported type for context keys defined in this package, @@ -92,3 +95,14 @@ func (r *InstrumentedRoundTripper) RoundTrip(req *http.Request) (*http.Response, RecordHTTPClientRequest(duration, resp.StatusCode, false, scaler, triggerName, metricName, namespace, scaledResource) return resp, nil } + +// BuildScalerRequestCtx attaches scaler metadata used by HTTP client +// instrumentation to the outbound request context. +func BuildScalerRequestCtx(ctx context.Context, config scalersconfig.ScalerConfig, metricName string) context.Context { + requestCtx := context.WithValue(ctx, ScalerContextKey, config.TriggerType) + requestCtx = context.WithValue(requestCtx, TriggerNameContextKey, config.TriggerName) + requestCtx = context.WithValue(requestCtx, MetricNameContextKey, metricName) + requestCtx = context.WithValue(requestCtx, NamespaceContextKey, config.ScalableObjectNamespace) + requestCtx = context.WithValue(requestCtx, ScaledResourceContextKey, config.ScalableObjectName) + return requestCtx +} diff --git a/pkg/metricscollector/request_context.go b/pkg/metricscollector/request_context.go deleted file mode 100644 index d775c91766f..00000000000 --- a/pkg/metricscollector/request_context.go +++ /dev/null @@ -1,34 +0,0 @@ -/* -Copyright 2026 The KEDA Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package metricscollector - -import ( - "context" - - "github.com/kedacore/keda/v2/pkg/scalers/scalersconfig" -) - -// BuildScalerRequestCtx attaches scaler metadata used by HTTP client -// instrumentation to the outbound request context. -func BuildScalerRequestCtx(ctx context.Context, config scalersconfig.ScalerConfig, metricName string) context.Context { - requestCtx := context.WithValue(ctx, ScalerContextKey, config.TriggerType) - requestCtx = context.WithValue(requestCtx, TriggerNameContextKey, config.TriggerName) - requestCtx = context.WithValue(requestCtx, MetricNameContextKey, metricName) - requestCtx = context.WithValue(requestCtx, NamespaceContextKey, config.ScalableObjectNamespace) - requestCtx = context.WithValue(requestCtx, ScaledResourceContextKey, config.ScalableObjectName) - return requestCtx -} diff --git a/pkg/scalers/aws/aws_sigv4_test.go b/pkg/scalers/aws/aws_sigv4_test.go index 9c49be40143..65421a9d5b3 100644 --- a/pkg/scalers/aws/aws_sigv4_test.go +++ b/pkg/scalers/aws/aws_sigv4_test.go @@ -36,9 +36,9 @@ import ( ) func TestSigV4RoundTripper(t *testing.T) { - transport := util.CreateRT(false) + roundtripper := util.CreateRT(false) - cli := &http.Client{Transport: transport} + cli := &http.Client{Transport: roundtripper} req, err := http.NewRequest(http.MethodGet, "https://aps-workspaces.us-west-2.amazonaws.com/workspaces/ws-38377ca8-8db3-4b58-812d-b65a81837bb8/api/v1/query?query=vector(10)", strings.NewReader("Hello, world!")) require.NoError(t, err) diff --git a/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go index 123d06f47c2..63f18bf9886 100644 --- a/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go +++ b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go @@ -55,9 +55,9 @@ func TryAndGetAzureManagedPrometheusHTTPRoundTripper(logger logr.Logger, podIden return nil, err } - transport := util.CreateRT(false) + roundtripper := util.CreateRT(false) rt := &azureManagedPrometheusHTTPRoundTripper{ - next: transport, + next: roundtripper, chainedCredential: chainedCred, resourceURL: resourceURLBasedOnCloud, } diff --git a/pkg/scalers/github_runner_scaler.go b/pkg/scalers/github_runner_scaler.go index 5086d44e157..8563baba81a 100644 --- a/pkg/scalers/github_runner_scaler.go +++ b/pkg/scalers/github_runner_scaler.go @@ -347,8 +347,8 @@ func NewGitHubRunnerScaler(config *scalersconfig.ScalerConfig) (Scaler, error) { } if meta.ApplicationID != 0 && meta.InstallationID != 0 && meta.ApplicationKey != "" { - httpTrans := kedautil.CreateRT(false) - hc, err := gha.New(httpTrans, meta.ApplicationID, meta.InstallationID, []byte(meta.ApplicationKey)) + roundtripper := kedautil.CreateRT(false) + hc, err := gha.New(roundtripper, meta.ApplicationID, meta.InstallationID, []byte(meta.ApplicationKey)) if err != nil { return nil, fmt.Errorf("error creating GitHub App client: %w, \n appID: %d, instID: %d", err, meta.ApplicationID, meta.InstallationID) } diff --git a/pkg/util/http_roundtripper_integration_test.go b/pkg/util/http_roundtripper_integration_test.go deleted file mode 100644 index fe1a30ab24e..00000000000 --- a/pkg/util/http_roundtripper_integration_test.go +++ /dev/null @@ -1,31 +0,0 @@ -/* -Copyright 2026 The KEDA Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "fmt" - "testing" - - . "github.com/onsi/gomega" -) - -func TestCreateHTTPClient_TransportIsInstrumented(t *testing.T) { - RegisterTestingT(t) - - client := CreateHTTPClient(0, false) - Expect(fmt.Sprintf("%T", client.Transport)).To(Equal("*metricscollector.InstrumentedRoundTripper")) -} From 39950af074bb5850fa8b3d029126fc5af27dce96 Mon Sep 17 00:00:00 2001 From: Ali Aqel Date: Thu, 23 Apr 2026 06:16:10 +0000 Subject: [PATCH 20/22] Fix http roundtripper test bodyclose issues Signed-off-by: Ali Aqel --- .../http_roundtripper_test.go | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/pkg/metricscollector/http_roundtripper_test.go b/pkg/metricscollector/http_roundtripper_test.go index 8d49da20205..df7d56ce086 100644 --- a/pkg/metricscollector/http_roundtripper_test.go +++ b/pkg/metricscollector/http_roundtripper_test.go @@ -79,7 +79,8 @@ func TestInstrumentedRoundTripper_RecordsSuccessfulResponses(t *testing.T) { metricName := fmt.Sprintf("metric-%d", tc.statusCode) scaledResource := fmt.Sprintf("so-%d", tc.statusCode) - rt := NewInstrumentedRoundTripper(&mockRoundTripper{resp: fakeResponse(tc.statusCode)}) + response := fakeResponse(tc.statusCode) + rt := NewInstrumentedRoundTripper(&mockRoundTripper{resp: response}) ctx := context.Background() ctx = context.WithValue(ctx, ScalerContextKey, scalerName) @@ -94,6 +95,8 @@ func TestInstrumentedRoundTripper_RecordsSuccessfulResponses(t *testing.T) { resp, err := rt.RoundTrip(req) Expect(err).To(BeNil()) Expect(resp).NotTo(BeNil()) + Expect(resp).To(Equal(response)) + defer resp.Body.Close() counterValue, err := httpClientRequestsTotal. GetMetricWithLabelValues("default", scaledResource, scalerName, triggerName, metricName, fmt.Sprintf("%d", tc.statusCode)) @@ -138,6 +141,9 @@ func TestInstrumentedRoundTripper_RecordsTransportErrors(t *testing.T) { Expect(err).To(BeNil()) resp, err := rt.RoundTrip(req) + if resp != nil { + defer resp.Body.Close() + } Expect(err).To(Equal(transportErr)) Expect(resp).To(BeNil()) @@ -162,15 +168,16 @@ func TestInstrumentedRoundTripper_RecordsTransportErrors(t *testing.T) { func TestInstrumentedRoundTripper_ResponseReturnedUnmodified(t *testing.T) { RegisterTestingT(t) - expected := fakeResponse(http.StatusAccepted) - rt := NewInstrumentedRoundTripper(&mockRoundTripper{resp: expected}) + response := fakeResponse(http.StatusAccepted) + rt := NewInstrumentedRoundTripper(&mockRoundTripper{resp: response}) req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, "http://example.com", nil) Expect(err).To(BeNil()) got, err := rt.RoundTrip(req) Expect(err).To(BeNil()) - Expect(got).To(Equal(expected)) + Expect(got).To(Equal(response)) + defer got.Body.Close() } func TestInstrumentedRoundTripper_NilNextUsesDefault(t *testing.T) { @@ -182,7 +189,8 @@ func TestInstrumentedRoundTripper_NilNextUsesDefault(t *testing.T) { func TestInstrumentedRoundTripper_ScalerContextKey_Missing(t *testing.T) { RegisterTestingT(t) - rt := NewInstrumentedRoundTripper(&mockRoundTripper{resp: fakeResponse(200)}) + response := fakeResponse(200) + rt := NewInstrumentedRoundTripper(&mockRoundTripper{resp: response}) req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, "http://example.com", nil) Expect(err).To(BeNil()) @@ -190,4 +198,6 @@ func TestInstrumentedRoundTripper_ScalerContextKey_Missing(t *testing.T) { resp, err := rt.RoundTrip(req) Expect(err).To(BeNil()) Expect(resp).NotTo(BeNil()) + Expect(resp).To(Equal(response)) + defer resp.Body.Close() } From 7b0abbf000aa380862fcb086db24c6a1f8597393 Mon Sep 17 00:00:00 2001 From: Ali Aqel Date: Thu, 23 Apr 2026 06:28:51 +0000 Subject: [PATCH 21/22] Rename roundtripper locals to rt Signed-off-by: Ali Aqel --- pkg/scalers/aws/aws_sigv4_test.go | 4 ++-- .../azure/azure_managed_prometheus_http_round_tripper.go | 8 ++++---- pkg/scalers/github_runner_scaler.go | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/scalers/aws/aws_sigv4_test.go b/pkg/scalers/aws/aws_sigv4_test.go index 65421a9d5b3..5b442988b87 100644 --- a/pkg/scalers/aws/aws_sigv4_test.go +++ b/pkg/scalers/aws/aws_sigv4_test.go @@ -36,9 +36,9 @@ import ( ) func TestSigV4RoundTripper(t *testing.T) { - roundtripper := util.CreateRT(false) + rt := util.CreateRT(false) - cli := &http.Client{Transport: roundtripper} + cli := &http.Client{Transport: rt} req, err := http.NewRequest(http.MethodGet, "https://aps-workspaces.us-west-2.amazonaws.com/workspaces/ws-38377ca8-8db3-4b58-812d-b65a81837bb8/api/v1/query?query=vector(10)", strings.NewReader("Hello, world!")) require.NoError(t, err) diff --git a/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go index 63f18bf9886..71e9e6a6c8e 100644 --- a/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go +++ b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go @@ -55,13 +55,13 @@ func TryAndGetAzureManagedPrometheusHTTPRoundTripper(logger logr.Logger, podIden return nil, err } - roundtripper := util.CreateRT(false) - rt := &azureManagedPrometheusHTTPRoundTripper{ - next: roundtripper, + rt := util.CreateRT(false) + managedPrometheusRT := &azureManagedPrometheusHTTPRoundTripper{ + next: rt, chainedCredential: chainedCred, resourceURL: resourceURLBasedOnCloud, } - return rt, nil + return managedPrometheusRT, nil } // Not azure managed prometheus. Don't create a round tripper and don't return error. diff --git a/pkg/scalers/github_runner_scaler.go b/pkg/scalers/github_runner_scaler.go index 8563baba81a..fb464c38320 100644 --- a/pkg/scalers/github_runner_scaler.go +++ b/pkg/scalers/github_runner_scaler.go @@ -347,8 +347,8 @@ func NewGitHubRunnerScaler(config *scalersconfig.ScalerConfig) (Scaler, error) { } if meta.ApplicationID != 0 && meta.InstallationID != 0 && meta.ApplicationKey != "" { - roundtripper := kedautil.CreateRT(false) - hc, err := gha.New(roundtripper, meta.ApplicationID, meta.InstallationID, []byte(meta.ApplicationKey)) + rt := kedautil.CreateRT(false) + hc, err := gha.New(rt, meta.ApplicationID, meta.InstallationID, []byte(meta.ApplicationKey)) if err != nil { return nil, fmt.Errorf("error creating GitHub App client: %w, \n appID: %d, instID: %d", err, meta.ApplicationID, meta.InstallationID) } From 111c39e9efb2e545107b3c2ff93c3ed371515a29 Mon Sep 17 00:00:00 2001 From: Ali Aqel Date: Thu, 23 Apr 2026 06:30:49 +0000 Subject: [PATCH 22/22] Rename HTTP client transport local to rt Signed-off-by: Ali Aqel --- pkg/util/http.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/util/http.go b/pkg/util/http.go index 538fb088754..bc74869c735 100644 --- a/pkg/util/http.go +++ b/pkg/util/http.go @@ -52,10 +52,10 @@ func CreateHTTPClient(timeout time.Duration, unsafeSsl bool) *http.Client { if timeout <= 0 { timeout = 300 * time.Millisecond } - transport := CreateRT(unsafeSsl) + rt := CreateRT(unsafeSsl) httpClient := &http.Client{ Timeout: timeout, - Transport: transport, + Transport: rt, } return httpClient }