add HTTP client request metrics for scaler metric requests

CHANGELOG.md

@@ -78,6 +78,7 @@ To learn more about active deprecations, we recommend checking [GitHub Discussio
 
 - **General**: Add CRD-level validation markers (Minimum, MinLength, MinItems, Enum) for ScaledObject, ScaledJob, ScaleTriggers, and TriggerAuthentication API types ([#7533](https://github.com/kedacore/keda/pull/7533))
 - **General**: Add `--leader-election-id` flag to allow configuring the leader election Lease name ([#7564](https://github.com/kedacore/keda/issues/7564))
+- **General**: Add scaler HTTP request metrics (`keda_scaler_http_requests_total`, `keda_scaler_http_request_duration_seconds`) for outbound HTTP requests made during scaler metric collection ([#6600](https://github.com/kedacore/keda/issues/6600))
 - **General**: Allow more control of TLS versions & ciphers via `KEDA_HTTP_TLS_CIPHER_LIST`, `KEDA_SERVICE_TLS_CIPHER_LIST` and `KEDA_SERVICE_MIN_TLS_VERSION` env vars ([#7617](https://github.com/kedacore/keda/pull/7617))
 - **General**: Make APIService cert injections optional ([#7559](https://github.com/kedacore/keda/pull/7559))
 - **AWS Scalers**: Add support for AWS External ID in TriggerAuthentication podIdentity for all AWS scalers (SQS, Kinesis, DynamoDB, CloudWatch, etc.) to enable cross-account access scenarios ([#6921](https://github.com/kedacore/keda/issues/6921))

pkg/metricscollector/http_roundtripper.go

@@ -0,0 +1,108 @@
+/*
+Copyright 2026 The KEDA Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metricscollector
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"github.com/kedacore/keda/v2/pkg/scalers/scalersconfig"
+)
+
+// contextKey is an unexported type for context keys defined in this package,
+// preventing collisions with keys from other packages.
+type contextKey string
+
+const (
+	// ScalerContextKey is the context key used to attach the scaler type name
+	// (e.g. "prometheus", "redis") to an outbound HTTP request so that metrics
+	// observers can include it as a dimension.
+	ScalerContextKey contextKey = "scaler"
+
+	// TriggerNameContextKey is the context key used to attach the user-defined
+	// trigger name to an outbound HTTP request.
+	TriggerNameContextKey contextKey = "trigger_name"
+
+	// MetricNameContextKey is the context key used to attach the metric name
+	// being queried to an outbound HTTP request.
+	MetricNameContextKey contextKey = "metric_name"
+
+	// NamespaceContextKey is the context key used to attach the namespace of the
+	// ScaledObject/ScaledJob that owns the scaler making the request.
+	NamespaceContextKey contextKey = "namespace"
+
+	// ScaledResourceContextKey is the context key used to attach the name of the
+	// ScaledObject/ScaledJob that owns the scaler making the request.
+	ScaledResourceContextKey contextKey = "scaled_resource"
+)
+
+// InstrumentedRoundTripper wraps an http.RoundTripper and records outbound
+// HTTP request metrics after every completed round-trip. It reads known
+// context keys from the request context to populate metric dimensions. It
+// does not buffer or inspect the response body.
+type InstrumentedRoundTripper struct {
+	next http.RoundTripper
+}
+
+// NewInstrumentedRoundTripper wraps next with a RoundTripper that records
+// HTTP request metrics after every request. If next is nil,
+// http.DefaultTransport is used.
+func NewInstrumentedRoundTripper(next http.RoundTripper) http.RoundTripper {
+	if next == nil {
+		next = http.DefaultTransport
+	}
+	return &InstrumentedRoundTripper{next: next}
+}
+
+func (r *InstrumentedRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	start := time.Now()
+	resp, err := r.next.RoundTrip(req)
+	duration := time.Since(start).Seconds()
+
+	ctx := req.Context()
+	scaler, scalerOK := ctx.Value(ScalerContextKey).(string)
+	triggerName, triggerOK := ctx.Value(TriggerNameContextKey).(string)
+	metricName, metricOK := ctx.Value(MetricNameContextKey).(string)
+	namespace, nsOK := ctx.Value(NamespaceContextKey).(string)
+	scaledResource, srOK := ctx.Value(ScaledResourceContextKey).(string)
+
+	// Only record metrics for scaler metric collection requests, identified by the
+	// presence of all five context keys injected by BuildScalerRequestCtx.
+	// Other HTTP calls (e.g. during scaler initialization) are not recorded.
+	if !scalerOK || !triggerOK || !metricOK || !nsOK || !srOK {
+		return resp, err
+	}
+
+	if err != nil {
+		RecordHTTPClientRequest(duration, 0, true, scaler, triggerName, metricName, namespace, scaledResource)
+		return nil, err
+	}
+	RecordHTTPClientRequest(duration, resp.StatusCode, false, scaler, triggerName, metricName, namespace, scaledResource)
+	return resp, nil
+}
+
+// BuildScalerRequestCtx attaches scaler metadata used by HTTP client
+// instrumentation to the outbound request context.
+func BuildScalerRequestCtx(ctx context.Context, config scalersconfig.ScalerConfig, metricName string) context.Context {
+	requestCtx := context.WithValue(ctx, ScalerContextKey, config.TriggerType)
+	requestCtx = context.WithValue(requestCtx, TriggerNameContextKey, config.TriggerName)
+	requestCtx = context.WithValue(requestCtx, MetricNameContextKey, metricName)
+	requestCtx = context.WithValue(requestCtx, NamespaceContextKey, config.ScalableObjectNamespace)
+	requestCtx = context.WithValue(requestCtx, ScaledResourceContextKey, config.ScalableObjectName)
+	return requestCtx
+}

pkg/metricscollector/http_roundtripper_test.go

@@ -0,0 +1,203 @@
+/*
+Copyright 2026 The KEDA Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metricscollector
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+
+	. "github.com/onsi/gomega"
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+)
+
+type mockRoundTripper struct {
+	resp *http.Response
+	err  error
+}
+
+func (m *mockRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	if m.resp != nil && m.resp.Request == nil {
+		m.resp.Request = req
+	}
+
+	return m.resp, m.err
+}
+
+func fakeResponse(statusCode int) *http.Response {
+	return &http.Response{
+		StatusCode: statusCode,
+		Body:       io.NopCloser(strings.NewReader("")),
+	}
+}
+
+func withPromCollector(t *testing.T) {
+	t.Helper()
+
+	previousCollectors := collectors
+	collectors = []MetricsCollector{&PromMetrics{}}
+	t.Cleanup(func() {
+		collectors = previousCollectors
+	})
+}
+
+func TestInstrumentedRoundTripper_RecordsSuccessfulResponses(t *testing.T) {
+	testCases := []struct {
+		name       string
+		statusCode int
+	}{
+		{name: "2xx response", statusCode: http.StatusOK},
+		{name: "4xx response", statusCode: http.StatusBadRequest},
+		{name: "5xx response", statusCode: http.StatusInternalServerError},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			RegisterTestingT(t)
+			withPromCollector(t)
+
+			scalerName := fmt.Sprintf("prometheus-%d", tc.statusCode)
+			triggerName := fmt.Sprintf("trigger-%d", tc.statusCode)
+			metricName := fmt.Sprintf("metric-%d", tc.statusCode)
+			scaledResource := fmt.Sprintf("so-%d", tc.statusCode)
+
+			response := fakeResponse(tc.statusCode)
+			rt := NewInstrumentedRoundTripper(&mockRoundTripper{resp: response})
+
+			ctx := context.Background()
+			ctx = context.WithValue(ctx, ScalerContextKey, scalerName)
+			ctx = context.WithValue(ctx, TriggerNameContextKey, triggerName)
+			ctx = context.WithValue(ctx, MetricNameContextKey, metricName)
+			ctx = context.WithValue(ctx, NamespaceContextKey, "default")
+			ctx = context.WithValue(ctx, ScaledResourceContextKey, scaledResource)
+
+			req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://example.com", nil)
+			Expect(err).To(BeNil())
+
+			resp, err := rt.RoundTrip(req)
+			Expect(err).To(BeNil())
+			Expect(resp).NotTo(BeNil())
+			Expect(resp).To(Equal(response))
+			defer resp.Body.Close()
+
+			counterValue, err := httpClientRequestsTotal.
+				GetMetricWithLabelValues("default", scaledResource, scalerName, triggerName, metricName, fmt.Sprintf("%d", tc.statusCode))
+			Expect(err).To(BeNil())
+			Expect(counterValue).NotTo(BeNil())
+			m := &dto.Metric{}
+			err = counterValue.Write(m)
+			Expect(err).To(BeNil())
+			Expect(m.Counter.GetValue()).To(BeNumerically("==", 1))
+
+			durationHistogram, err := httpClientRequestDuration.
+				GetMetricWithLabelValues(scalerName, fmt.Sprintf("%d", tc.statusCode))
+			Expect(err).To(BeNil())
+			Expect(durationHistogram).NotTo(BeNil())
+			err = durationHistogram.(prometheus.Metric).Write(m)
+			Expect(err).To(BeNil())
+			Expect(m.Histogram.GetSampleCount()).To(BeNumerically("==", 1))
+		})
+	}
+}
+
+func TestInstrumentedRoundTripper_RecordsTransportErrors(t *testing.T) {
+	RegisterTestingT(t)
+	withPromCollector(t)
+
+	scalerName := "prometheus-transport-error"
+	triggerName := "trigger-transport-error"
+	metricName := "metric-transport-error"
+	scaledResource := "so-transport-error"
+
+	transportErr := io.ErrUnexpectedEOF
+	rt := NewInstrumentedRoundTripper(&mockRoundTripper{err: transportErr})
+
+	ctx := context.Background()
+	ctx = context.WithValue(ctx, ScalerContextKey, scalerName)
+	ctx = context.WithValue(ctx, TriggerNameContextKey, triggerName)
+	ctx = context.WithValue(ctx, MetricNameContextKey, metricName)
+	ctx = context.WithValue(ctx, NamespaceContextKey, "default")
+	ctx = context.WithValue(ctx, ScaledResourceContextKey, scaledResource)
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://example.com", nil)
+	Expect(err).To(BeNil())
+
+	resp, err := rt.RoundTrip(req)
+	if resp != nil {
+		defer resp.Body.Close()
+	}
+	Expect(err).To(Equal(transportErr))
+	Expect(resp).To(BeNil())
+
+	counterValue, err := httpClientRequestsTotal.
+		GetMetricWithLabelValues("default", scaledResource, scalerName, triggerName, metricName, "error")
+	Expect(err).To(BeNil())
+	Expect(counterValue).NotTo(BeNil())
+	m := &dto.Metric{}
+	err = counterValue.Write(m)
+	Expect(err).To(BeNil())
+	Expect(m.Counter.GetValue()).To(BeNumerically("==", 1))
+
+	durationHistogram, err := httpClientRequestDuration.
+		GetMetricWithLabelValues(scalerName, "error")
+	Expect(err).To(BeNil())
+	Expect(durationHistogram).NotTo(BeNil())
+	err = durationHistogram.(prometheus.Metric).Write(m)
+	Expect(err).To(BeNil())
+	Expect(m.Histogram.GetSampleCount()).To(BeNumerically("==", 1))
+}
+
+func TestInstrumentedRoundTripper_ResponseReturnedUnmodified(t *testing.T) {
+	RegisterTestingT(t)
+
+	response := fakeResponse(http.StatusAccepted)
+	rt := NewInstrumentedRoundTripper(&mockRoundTripper{resp: response})
+
+	req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, "http://example.com", nil)
+	Expect(err).To(BeNil())
+
+	got, err := rt.RoundTrip(req)
+	Expect(err).To(BeNil())
+	Expect(got).To(Equal(response))
+	defer got.Body.Close()
+}
+
+func TestInstrumentedRoundTripper_NilNextUsesDefault(t *testing.T) {
+	RegisterTestingT(t)
+
+	rt := NewInstrumentedRoundTripper(nil)
+	Expect(fmt.Sprintf("%T", rt)).To(Equal("*metricscollector.InstrumentedRoundTripper"))
+}
+
+func TestInstrumentedRoundTripper_ScalerContextKey_Missing(t *testing.T) {
+	RegisterTestingT(t)
+	response := fakeResponse(200)
+	rt := NewInstrumentedRoundTripper(&mockRoundTripper{resp: response})
+
+	req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, "http://example.com", nil)
+	Expect(err).To(BeNil())
+
+	resp, err := rt.RoundTrip(req)
+	Expect(err).To(BeNil())
+	Expect(resp).NotTo(BeNil())
+	Expect(resp).To(Equal(response))
+	defer resp.Body.Close()
+}

pkg/metricscollector/metricscollectors.go

@@ -17,6 +17,7 @@ limitations under the License.
 package metricscollector
 
 import (
+	"strconv"
 	"time"
 
 	grpcprom "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus"
@@ -79,6 +80,12 @@ type MetricsCollector interface {
 
 	// RecordCloudEventQueueStatus record the number of cloudevents that are waiting for emitting
 	RecordCloudEventQueueStatus(namespace string, value int)
+
+	// RecordHTTPClientRequest records the duration and outcome of an outbound HTTP request
+	// made by one of KEDA's internal HTTP clients. scaler, triggerName, metricName,
+	// namespace, and scaledResource are provided explicitly by the caller; upstream
+	// instrumentation may derive them from context before invoking the collector.
+	RecordHTTPClientRequest(durationSeconds float64, statusCode int, isError bool, scaler, triggerName, metricName, namespace, scaledResource string)
 }
 
 func NewMetricsCollectors(enablePrometheusMetrics bool, enableOpenTelemetryMetrics bool) {
@@ -205,6 +212,22 @@ func RecordCloudEventQueueStatus(namespace string, value int) {
 	}
 }
 
+// RecordHTTPClientRequest records the duration and outcome of an outbound HTTP request
+// made by one of KEDA's internal HTTP clients. Called by InstrumentedRoundTripper in
+// the util package after extracting label values from the request context.
+func RecordHTTPClientRequest(durationSeconds float64, statusCode int, isError bool, scaler, triggerName, metricName, namespace, scaledResource string) {
+	for _, element := range collectors {
+		element.RecordHTTPClientRequest(durationSeconds, statusCode, isError, scaler, triggerName, metricName, namespace, scaledResource)
+	}
+}
+
+func httpStatusCodeLabel(code int, isError bool) string {
+	if isError {
+		return "error"
+	}
+	return strconv.Itoa(code)
+}
+
 // Returns the ServerMetrics object for GRPC Server metrics. Used to initialize the GRPC server with the proper intercepts
 // Currently, only Prometheus metrics are supported.
 func GetServerMetrics() *grpcprom.ServerMetrics {