chore(deps): update node exporter to 1.12.1

e2e/nodeexporter/metrics.go

@@ -2,23 +2,20 @@ package nodeexporter
 
 import (
 	"fmt"
+	"regexp"
+	"strconv"
 	"strings"
 )
 
+var (
+	metricNamePattern          = regexp.MustCompile(`^([^{ \t]+)(?:\{[^}]*\})?[ \t]+`)
+	nodeExporterVersionPattern = regexp.MustCompile(`(?m)^node_exporter_build_info\{[^\n}]*version="v([0-9]+)\.([0-9]+)\.[^"}]+"[^\n}]*\} 1$`)
+)
+
 // ValidateMetrics verifies that a node-exporter scrape contains every metric
 // retained by the AKS default Prometheus profile.
 func ValidateMetrics(metricsText string) error {
-	metricNames := make(map[string]struct{})
-	for line := range strings.SplitSeq(metricsText, "\n") {
-		line = strings.TrimSpace(line)
-		if line == "" || strings.HasPrefix(line, "#") {
-			continue
-		}
-
-		if end := strings.IndexAny(line, "{ \t"); end > 0 {
-			metricNames[line[:end]] = struct{}{}
-		}
-	}
+	metricNames := parseMetricNames(metricsText)
 
 	requiredMetrics := []string{
 		"node_disk_read_time_seconds_total",
@@ -49,3 +46,93 @@ func ValidateMetrics(metricsText string) error {
 
 	return nil
 }
+
+func parseMetricNames(metricsText string) map[string]struct{} {
+	metricNames := make(map[string]struct{})
+	for line := range strings.SplitSeq(metricsText, "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+
+		if match := metricNamePattern.FindStringSubmatch(line); len(match) == 2 {
+			metricNames[match[1]] = struct{}{}
+		}
+	}
+	return metricNames
+}
+
+// ValidateCollectors verifies that the collectors enabled by AgentBaker are present in the scrape.
+// InfiniBand metrics are required only when the node has InfiniBand hardware.
+func ValidateCollectors(metricsText string, requireInfiniBand bool) error {
+	isVersion112OrNewer, err := nodeExporterVersionAtLeast(metricsText, 1, 12)
+	if err != nil {
+		return err
+	}
+	if isVersion112OrNewer {
+		requiredCollectors := []string{"bcachefs", "dmmultipath", "kernel_hung"}
+		var missingCollectors []string
+		for _, collector := range requiredCollectors {
+			if !strings.Contains(metricsText, `node_scrape_collector_success{collector="`+collector+`"}`) {
+				missingCollectors = append(missingCollectors, collector)
+			}
+		}
+		if len(missingCollectors) > 0 {
+			return fmt.Errorf("enabled collectors are missing: %s", strings.Join(missingCollectors, ", "))
+		}
+	}
+
+	if requireInfiniBand {
+		if !strings.Contains(metricsText, `node_scrape_collector_success{collector="infiniband"} 1`) {
+			return fmt.Errorf("InfiniBand collector did not succeed")
+		}
+		hasInfiniBandMetric := false
+		for name := range parseMetricNames(metricsText) {
+			if strings.HasPrefix(name, "node_infiniband_") {
+				hasInfiniBandMetric = true
+				break
+			}
+		}
+		if !hasInfiniBandMetric {
+			return fmt.Errorf("InfiniBand metrics are missing")
+		}
+
+		if isVersion112OrNewer {
+			hardwareCounterMetrics := []string{
+				"node_infiniband_duplicate_requests_packets_total",
+				"node_infiniband_lifespan_seconds",
+				"node_infiniband_out_of_buffer_drops_total",
+				"node_infiniband_rx_write_requests_total",
+			}
+			hasHardwareCounterMetric := false
+			metricNames := parseMetricNames(metricsText)
+			for _, name := range hardwareCounterMetrics {
+				if _, exists := metricNames[name]; exists {
+					hasHardwareCounterMetric = true
+					break
+				}
+			}
+			if !hasHardwareCounterMetric {
+				return fmt.Errorf("InfiniBand hardware counter metrics are missing")
+			}
+		}
+	}
+
+	return nil
+}
+
+func nodeExporterVersionAtLeast(metricsText string, requiredMajor, requiredMinor int) (bool, error) {
+	match := nodeExporterVersionPattern.FindStringSubmatch(metricsText)
+	if len(match) != 3 {
+		return false, fmt.Errorf("node exporter build info is missing a valid version")
+	}
+	major, err := strconv.Atoi(match[1])
+	if err != nil {
+		return false, fmt.Errorf("parse node exporter major version: %w", err)
+	}
+	minor, err := strconv.Atoi(match[2])
+	if err != nil {
+		return false, fmt.Errorf("parse node exporter minor version: %w", err)
+	}
+	return major > requiredMajor || major == requiredMajor && minor >= requiredMinor, nil
+}

e2e/nodeexporter/metrics_test.go

@@ -23,6 +23,57 @@ func TestValidateMetricsRejectsMissingMetric(t *testing.T) {
 	require.ErrorContains(t, err, "node_network_receive_bytes_total")
 }
 
+func TestValidateCollectors(t *testing.T) {
+	metrics := `node_exporter_build_info{version="v1.12.1"} 1
+node_scrape_collector_success{collector="bcachefs"} 0
+node_scrape_collector_success{collector="dmmultipath"} 1
+node_scrape_collector_success{collector="kernel_hung"} 0
+`
+
+	require.NoError(t, ValidateCollectors(metrics, false))
+}
+
+func TestValidateCollectorsRejectsMissingCollector(t *testing.T) {
+	metrics := `node_exporter_build_info{version="v1.12.1"} 1
+node_scrape_collector_success{collector="bcachefs"} 0
+node_scrape_collector_success{collector="kernel_hung"} 0
+`
+
+	require.ErrorContains(t, ValidateCollectors(metrics, false), "dmmultipath")
+}
+
+func TestValidateCollectorsRequiresInfiniBandMetricsOnInfiniBandNodes(t *testing.T) {
+	metrics := `node_exporter_build_info{version="v1.12.1"} 1
+node_scrape_collector_success{collector="bcachefs"} 0
+node_scrape_collector_success{collector="dmmultipath"} 1
+node_scrape_collector_success{collector="kernel_hung"} 0
+node_scrape_collector_success{collector="infiniband"} 1
+node_infiniband_port_state{device="mlx5_0",port="1",state="ACTIVE"} 1
+node_infiniband_out_of_buffer_drops_total{device="mlx5_0",port="1"} 0
+`
+
+	require.NoError(t, ValidateCollectors(metrics, true))
+	require.ErrorContains(t, ValidateCollectors(strings.Replace(metrics, "node_infiniband_", "other_metric_", -1), true), "metrics are missing")
+	require.ErrorContains(t, ValidateCollectors(strings.Replace(metrics, "\nnode_infiniband_out_of_buffer_drops_total", "\nother_metric", 1), true), "hardware counter metrics are missing")
+	require.ErrorContains(t, ValidateCollectors(strings.Replace(metrics, `collector="infiniband"} 1`, `collector="infiniband"} 0`, 1), true), "did not succeed")
+}
+
+func TestValidateCollectorsSupportsMainVHDExporter(t *testing.T) {
+	metrics := `node_exporter_build_info{version="v1.9.1"} 1`
+
+	require.NoError(t, ValidateCollectors(metrics, false))
+}
+
+func TestValidateCollectorsRequiresNewCollectorsAfter112(t *testing.T) {
+	metrics := `node_exporter_build_info{version="v1.13.0"} 1`
+
+	require.ErrorContains(t, ValidateCollectors(metrics, false), "bcachefs")
+}
+
+func TestValidateCollectorsRejectsMissingBuildInfo(t *testing.T) {
+	require.ErrorContains(t, ValidateCollectors("", false), "build info")
+}
+
 func validMetrics() string {
 	return `# TYPE node_disk_read_time_seconds_total counter
 node_disk_read_time_seconds_total{device="sda"} 1

e2e/validators.go

@@ -2827,7 +2827,11 @@ func ValidateNodeExporter(ctx context.Context, s *Scenario) error {
 	// so this also verifies that the endpoint is reachable on the address used by monitoring infrastructure.
 	s.Logger.Logf("Validating node-exporter metrics on port 19100")
 	metricsURL := fmt.Sprintf("http://%s:19100/metrics", s.Runtime.VM.PrivateIP)
-	errs = append(errs, scrapeAndValidateNodeExporter(ctx, s, metricsURL))
+	hasInfiniBand, err := nodeHasInfiniBandHardware(ctx, s)
+	if err != nil {
+		errs = append(errs, err)
+	}
+	errs = append(errs, scrapeAndValidateNodeExporter(ctx, s, metricsURL, hasInfiniBand))
 
 	if _, err := execScriptOnVMForScenarioValidateExitCode(ctx, s, fmt.Sprintf("systemctl is-active %s", serviceName), 0,
 		"node-exporter should remain active after scraping"); err != nil {
@@ -2841,7 +2845,16 @@ func ValidateNodeExporter(ctx context.Context, s *Scenario) error {
 	return nil
 }
 
-func scrapeAndValidateNodeExporter(ctx context.Context, s *Scenario, metricsURL string) error {
+func nodeHasInfiniBandHardware(ctx context.Context, s *Scenario) (bool, error) {
+	command := `for device in /sys/class/infiniband/*; do [ -e "$device" ] && exit 0; done; exit 1`
+	result, err := execScriptOnVMForScenario(ctx, s, command)
+	if err != nil {
+		return false, fmt.Errorf("detect InfiniBand hardware: %w", err)
+	}
+	return result.exitCode == "0", nil
+}
+
+func scrapeAndValidateNodeExporter(ctx context.Context, s *Scenario, metricsURL string, requireInfiniBand bool) error {
 	result, err := execScriptOnVMForScenario(ctx, s, fmt.Sprintf("curl --noproxy '*' -sS --max-time 10 %q", metricsURL))
 	if err != nil {
 		return fmt.Errorf("scrape node-exporter metrics from %s: %w", metricsURL, err)
@@ -2856,7 +2869,10 @@ func scrapeAndValidateNodeExporter(ctx context.Context, s *Scenario, metricsURL
 	if len(responsePreview) > previewLimit {
 		responsePreview = responsePreview[:previewLimit] + "\n... response truncated"
 	}
-	return assert.NoError(nodeexporter.ValidateMetrics(result.stdout), "node-exporter scrape did not satisfy the AKS Prometheus metrics contract\nresponse preview:\n%s", responsePreview)
+	return errors.Join(
+		assert.NoError(nodeexporter.ValidateMetrics(result.stdout), "node-exporter scrape did not satisfy the AKS Prometheus metrics contract\nresponse preview:\n%s", responsePreview),
+		assert.NoError(nodeexporter.ValidateCollectors(result.stdout, requireInfiniBand), "node-exporter collectors did not satisfy the AgentBaker contract\nresponse preview:\n%s", responsePreview),
+	)
 }
 
 func ValidateNPDFilesystemCorruption(ctx context.Context, s *Scenario) (err error) {

parts/common/components.json

@@ -2183,31 +2183,31 @@
             "versionsV2": [
               {
                 "renovateTag": "name=node-exporter-kubernetes, repository=production, os=ubuntu, release=26.04",
-                "latestVersion": "1.9.1-ubuntu26.04u28"
+                "latestVersion": "1.12.1-ubuntu26.04u7"
               }
             ]
           },
           "r2404": {
             "versionsV2": [
               {
                 "renovateTag": "name=node-exporter-kubernetes, repository=production, os=ubuntu, release=24.04",
-                "latestVersion": "1.9.1-ubuntu24.04u28"
+                "latestVersion": "1.12.1-ubuntu24.04u7"
               }
             ]
           },
           "r2204": {
             "versionsV2": [
               {
                 "renovateTag": "name=node-exporter-kubernetes, repository=production, os=ubuntu, release=22.04",
-                "latestVersion": "1.9.1-ubuntu22.04u28"
+                "latestVersion": "1.12.1-ubuntu22.04u7"
               }
             ]
           },
           "r2004": {
             "versionsV2": [
               {
                 "renovateTag": "name=node-exporter-kubernetes, repository=production, os=ubuntu, release=20.04",
-                "latestVersion": "1.9.1-ubuntu20.04u28"
+                "latestVersion": "1.12.1-ubuntu20.04u7"
               }
             ]
           }
@@ -2217,7 +2217,7 @@
             "versionsV2": [
               {
                 "renovateTag": "RPM_registry=https://packages.microsoft.com/azurelinux/3.0/prod/cloud-native/x86_64/repodata, name=node-exporter-kubernetes, os=azurelinux, release=3.0",
-                "latestVersion": "1.9.1-28.azl3"
+                "latestVersion": "1.12.1-7.azl3"
               }
             ]
           },

vhdbuilder/packer/test/linux-vhd-content-test.sh

@@ -1993,6 +1993,22 @@ testNodeExporter () {
   fi
   echo "$test: skip sentinel file exists at $skip_file"
 
+  local expectedVersion
+  expectedVersion=$(getPackageExpectedVersion "node-exporter" "" "" "")
+  if [ "$expectedVersion" = "<SKIP>" ]; then
+    err "$test" "node-exporter expected version is <SKIP> on supported OS $os_sku"
+    return 1
+  fi
+  assertPackageVersion "$test" "node-exporter-kubernetes" "$expectedVersion" || return 1
+
+  local expectedBinaryVersion="v${expectedVersion%%-*}"
+  local binaryVersion
+  binaryVersion=$(/usr/bin/node-exporter --version 2>&1 | awk 'NR == 1 { print $3 }')
+  if [ "$binaryVersion" != "$expectedBinaryVersion" ]; then
+    err "$test" "node-exporter binary version '$binaryVersion' does not match expected '$expectedBinaryVersion'"
+    return 1
+  fi
+
   # The Dalec-built deb/rpm installs the binary to /usr/bin/node-exporter.
   # We then create a symlink at /opt/bin/node-exporter for consistency with
   # other binaries (kubelet, kubectl) that live in /opt/bin.