chore(deps): update node exporter to 1.12.1

e2e/nodeexporter/metrics.go

@@ -2,23 +2,20 @@ package nodeexporter
 
 import (
 	"fmt"
+	"regexp"
+	"strconv"
 	"strings"
 )
 
+var (
+	metricNamePattern          = regexp.MustCompile(`^([^{ \t]+)(?:\{[^}]*\})?[ \t]+`)
+	nodeExporterVersionPattern = regexp.MustCompile(`(?m)^node_exporter_build_info\{[^\n}]*version="v([0-9]+)\.([0-9]+)\.[^"}]+"[^\n}]*\} 1$`)
+)
+
 // ValidateMetrics verifies that a node-exporter scrape contains every metric
 // retained by the AKS default Prometheus profile.
 func ValidateMetrics(metricsText string) error {
-	metricNames := make(map[string]struct{})
-	for line := range strings.SplitSeq(metricsText, "\n") {
-		line = strings.TrimSpace(line)
-		if line == "" || strings.HasPrefix(line, "#") {
-			continue
-		}
-
-		if end := strings.IndexAny(line, "{ \t"); end > 0 {
-			metricNames[line[:end]] = struct{}{}
-		}
-	}
+	metricNames := parseMetricNames(metricsText)
 
 	requiredMetrics := []string{
 		"node_disk_read_time_seconds_total",
@@ -49,3 +46,79 @@ func ValidateMetrics(metricsText string) error {
 
 	return nil
 }
+
+func parseMetricNames(metricsText string) map[string]struct{} {
+	metricNames := make(map[string]struct{})
+	for line := range strings.SplitSeq(metricsText, "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+
+		if match := metricNamePattern.FindStringSubmatch(line); len(match) == 2 {
+			metricNames[match[1]] = struct{}{}
+		}
+	}
+	return metricNames
+}
+
+// ValidateCollectors verifies that the collectors enabled by AgentBaker are present in the scrape.
+// InfiniBand metrics are required only when the node has InfiniBand hardware.
+func ValidateCollectors(metricsText string, requireInfiniBand, requireInfiniBandDisabled bool) error {
+	isVersion112OrNewer, err := nodeExporterVersionAtLeast(metricsText, 1, 12)
+	if err != nil {
+		return err
+	}
+	if requireInfiniBandDisabled && strings.Contains(metricsText, `node_scrape_collector_success{collector="infiniband"}`) {
+		return fmt.Errorf("InfiniBand collector is enabled despite the MANA workaround")
+	}
+	if isVersion112OrNewer {
+		requiredCollectors := []string{"bcachefs", "dmmultipath", "kernel_hung"}
+		var missingCollectors []string
+		for _, collector := range requiredCollectors {
+			if !strings.Contains(metricsText, `node_scrape_collector_success{collector="`+collector+`"}`) {
+				missingCollectors = append(missingCollectors, collector)
+			}
+		}
+		if len(missingCollectors) > 0 {
+			return fmt.Errorf("enabled collectors are missing: %s", strings.Join(missingCollectors, ", "))
+		}
+	}
+
+	if requireInfiniBand {
+		if !strings.Contains(metricsText, `node_scrape_collector_success{collector="infiniband"} 1`) {
+			return fmt.Errorf("InfiniBand collector did not succeed")
+		}
+		hasInfiniBandMetric := false
+		for name := range parseMetricNames(metricsText) {
+			if strings.HasPrefix(name, "node_infiniband_") {
+				hasInfiniBandMetric = true
+				break
+			}
+		}
+		if !hasInfiniBandMetric {
+			return fmt.Errorf("InfiniBand metrics are missing")
+		}
+
+		// hw_counters are optional and driver-specific, even with exporter >=1.12.
+		// Successful collection must not require counters absent from the hardware.
+	}
+
+	return nil
+}
+
+func nodeExporterVersionAtLeast(metricsText string, requiredMajor, requiredMinor int) (bool, error) {
+	match := nodeExporterVersionPattern.FindStringSubmatch(metricsText)
+	if len(match) != 3 {
+		return false, fmt.Errorf("node exporter build info is missing a valid version")
+	}
+	major, err := strconv.Atoi(match[1])
+	if err != nil {
+		return false, fmt.Errorf("parse node exporter major version: %w", err)
+	}
+	minor, err := strconv.Atoi(match[2])
+	if err != nil {
+		return false, fmt.Errorf("parse node exporter minor version: %w", err)
+	}
+	return major > requiredMajor || major == requiredMajor && minor >= requiredMinor, nil
+}

e2e/nodeexporter/metrics_test.go

@@ -23,6 +23,72 @@ func TestValidateMetricsRejectsMissingMetric(t *testing.T) {
 	require.ErrorContains(t, err, "node_network_receive_bytes_total")
 }
 
+func TestValidateCollectors(t *testing.T) {
+	metrics := `node_exporter_build_info{version="v1.12.1"} 1
+node_scrape_collector_success{collector="bcachefs"} 0
+node_scrape_collector_success{collector="dmmultipath"} 1
+node_scrape_collector_success{collector="kernel_hung"} 0
+`
+
+	require.NoError(t, ValidateCollectors(metrics, false, false))
+}
+
+func TestValidateCollectorsRejectsMissingCollector(t *testing.T) {
+	metrics := `node_exporter_build_info{version="v1.12.1"} 1
+node_scrape_collector_success{collector="bcachefs"} 0
+node_scrape_collector_success{collector="kernel_hung"} 0
+`
+
+	require.ErrorContains(t, ValidateCollectors(metrics, false, false), "dmmultipath")
+}
+
+func TestValidateCollectorsRequiresInfiniBandMetricsOnInfiniBandNodes(t *testing.T) {
+	metrics := `node_exporter_build_info{version="v1.12.1"} 1
+node_scrape_collector_success{collector="bcachefs"} 0
+node_scrape_collector_success{collector="dmmultipath"} 1
+node_scrape_collector_success{collector="kernel_hung"} 0
+node_scrape_collector_success{collector="infiniband"} 1
+node_infiniband_state_id{device="mlx5_0",port="1"} 4
+node_infiniband_out_of_buffer_drops_total{device="mlx5_0",port="1"} 0
+`
+
+	require.NoError(t, ValidateCollectors(metrics, true, false))
+	require.ErrorContains(t, ValidateCollectors(strings.Replace(metrics, "node_infiniband_", "other_metric_", -1), true, false), "metrics are missing")
+	// A device without optional hw_counters still satisfies the common contract.
+	require.NoError(t, ValidateCollectors(strings.Replace(metrics, "node_infiniband_out_of_buffer_drops_total{device=\"mlx5_0\",port=\"1\"} 0\n", "", 1), true, false))
+	require.ErrorContains(t, ValidateCollectors(strings.Replace(metrics, `collector="infiniband"} 1`, `collector="infiniband"} 0`, 1), true, false), "did not succeed")
+}
+
+func TestValidateCollectorsSupportsMainVHDExporter(t *testing.T) {
+	metrics := `node_exporter_build_info{version="v1.9.1"} 1`
+
+	require.NoError(t, ValidateCollectors(metrics, false, false))
+}
+
+func TestValidateCollectorsRequiresNewCollectorsAfter112(t *testing.T) {
+	metrics := `node_exporter_build_info{version="v1.13.0"} 1`
+
+	require.ErrorContains(t, ValidateCollectors(metrics, false, false), "bcachefs")
+}
+
+func TestValidateCollectorsRejectsMissingBuildInfo(t *testing.T) {
+	require.ErrorContains(t, ValidateCollectors("", false, false), "build info")
+}
+
+func TestValidateCollectorsRequiresMANASuppression(t *testing.T) {
+	metrics := `node_exporter_build_info{version="v1.12.1"} 1
+node_scrape_collector_success{collector="bcachefs"} 0
+node_scrape_collector_success{collector="dmmultipath"} 1
+node_scrape_collector_success{collector="kernel_hung"} 0
+`
+	require.NoError(t, ValidateCollectors(metrics, false, true))
+	for _, value := range []string{"0", "1"} {
+		t.Run(value, func(t *testing.T) {
+			require.ErrorContains(t, ValidateCollectors(metrics+`node_scrape_collector_success{collector="infiniband"} `+value, false, true), "enabled despite the MANA workaround")
+		})
+	}
+}
+
 func validMetrics() string {
 	return `# TYPE node_disk_read_time_seconds_total counter
 node_disk_read_time_seconds_total{device="sda"} 1

e2e/validators.go

@@ -2827,7 +2827,11 @@ func ValidateNodeExporter(ctx context.Context, s *Scenario) error {
 	// so this also verifies that the endpoint is reachable on the address used by monitoring infrastructure.
 	s.Logger.Logf("Validating node-exporter metrics on port 19100")
 	metricsURL := fmt.Sprintf("http://%s:19100/metrics", s.Runtime.VM.PrivateIP)
-	errs = append(errs, scrapeAndValidateNodeExporter(ctx, s, metricsURL))
+	hasInfiniBand, err := nodeHasInfiniBandHardware(ctx, s)
+	if err != nil {
+		errs = append(errs, err)
+	}
+	errs = append(errs, scrapeAndValidateNodeExporter(ctx, s, metricsURL, hasInfiniBand))
 
 	if _, err := execScriptOnVMForScenarioValidateExitCode(ctx, s, fmt.Sprintf("systemctl is-active %s", serviceName), 0,
 		"node-exporter should remain active after scraping"); err != nil {
@@ -2841,7 +2845,49 @@ func ValidateNodeExporter(ctx context.Context, s *Scenario) error {
 	return nil
 }
 
-func scrapeAndValidateNodeExporter(ctx context.Context, s *Scenario, metricsURL string) error {
+func nodeHasInfiniBandHardware(ctx context.Context, s *Scenario) (bool, error) {
+	// node-exporter 1.12.1 cannot parse MANA RDMA devices, so the startup script
+	// disables the collector once MANA PCI hardware is observed this boot. Keep this
+	// detection aligned so MANA and mixed-HCA nodes do not require metrics from a
+	// collector that must be disabled pending an upstream fix.
+	command := `if [ -f /run/node-exporter-mana-observed ]; then
+    echo "MANA observed earlier this boot; InfiniBand collector disabled"
+    exit 1
+fi
+for device in /sys/bus/pci/devices/*; do
+    if [ -d "$device" ] &&
+       grep -qi '^0x1414$' "$device/vendor" 2>/dev/null &&
+       grep -Eqi '^0x00(b9|ba|c1)$' "$device/device" 2>/dev/null; then
+        echo "MANA PCI device: $device; InfiniBand collector disabled"
+        exit 1
+    fi
+done
+for device in /sys/class/infiniband/*; do
+    [ -e "$device" ] || continue
+    echo "InfiniBand device: $device; requiring collector success and metrics"
+    exit 0
+done
+echo "No InfiniBand devices found"
+exit 1`
+	result, err := execScriptOnVMForScenario(ctx, s, command)
+	if err != nil {
+		return false, fmt.Errorf("detect InfiniBand hardware: %w", err)
+	}
+	if result.exitCode != "0" && result.exitCode != "1" {
+		return false, fmt.Errorf("detect InfiniBand hardware: exit %s: %s", result.exitCode, result.stderr)
+	}
+	s.Logger.Logf("node-exporter hardware detection: %s", strings.TrimSpace(result.stdout))
+	return result.exitCode == "0", nil
+}
+
+func scrapeAndValidateNodeExporter(ctx context.Context, s *Scenario, metricsURL string, requireInfiniBand bool) error {
+	// The boot-local marker only exists on VHDs with lifecycle-aware suppression;
+	// main/older VHDs used by standalone PR E2Es retain their existing checks.
+	manaObserved, err := fileExist(ctx, s, "/run/node-exporter-mana-observed")
+	if err != nil {
+		return fmt.Errorf("read node-exporter MANA workaround state: %w", err)
+	}
+	s.Logger.Logf("node-exporter InfiniBand expectations: required=%t, disabled=%t", requireInfiniBand, manaObserved)
 	result, err := execScriptOnVMForScenario(ctx, s, fmt.Sprintf("curl --noproxy '*' -sS --max-time 10 %q", metricsURL))
 	if err != nil {
 		return fmt.Errorf("scrape node-exporter metrics from %s: %w", metricsURL, err)
@@ -2856,7 +2902,10 @@ func scrapeAndValidateNodeExporter(ctx context.Context, s *Scenario, metricsURL
 	if len(responsePreview) > previewLimit {
 		responsePreview = responsePreview[:previewLimit] + "\n... response truncated"
 	}
-	return assert.NoError(nodeexporter.ValidateMetrics(result.stdout), "node-exporter scrape did not satisfy the AKS Prometheus metrics contract\nresponse preview:\n%s", responsePreview)
+	return errors.Join(
+		assert.NoError(nodeexporter.ValidateMetrics(result.stdout), "node-exporter scrape did not satisfy the AKS Prometheus metrics contract\nresponse preview:\n%s", responsePreview),
+		assert.NoError(nodeexporter.ValidateCollectors(result.stdout, requireInfiniBand, manaObserved), "node-exporter collectors did not satisfy the AgentBaker contract\nresponse preview:\n%s", responsePreview),
+	)
 }
 
 func ValidateNPDFilesystemCorruption(ctx context.Context, s *Scenario) (err error) {

parts/common/components.json

@@ -2183,31 +2183,31 @@
             "versionsV2": [
               {
                 "renovateTag": "name=node-exporter-kubernetes, repository=production, os=ubuntu, release=26.04",
-                "latestVersion": "1.9.1-ubuntu26.04u28"
+                "latestVersion": "1.12.1-ubuntu26.04u7"
               }
             ]
           },
           "r2404": {
             "versionsV2": [
               {
                 "renovateTag": "name=node-exporter-kubernetes, repository=production, os=ubuntu, release=24.04",
-                "latestVersion": "1.9.1-ubuntu24.04u28"
+                "latestVersion": "1.12.1-ubuntu24.04u7"
               }
             ]
           },
           "r2204": {
             "versionsV2": [
               {
                 "renovateTag": "name=node-exporter-kubernetes, repository=production, os=ubuntu, release=22.04",
-                "latestVersion": "1.9.1-ubuntu22.04u28"
+                "latestVersion": "1.12.1-ubuntu22.04u7"
               }
             ]
           },
           "r2004": {
             "versionsV2": [
               {
                 "renovateTag": "name=node-exporter-kubernetes, repository=production, os=ubuntu, release=20.04",
-                "latestVersion": "1.9.1-ubuntu20.04u28"
+                "latestVersion": "1.12.1-ubuntu20.04u7"
               }
             ]
           }
@@ -2217,7 +2217,7 @@
             "versionsV2": [
               {
                 "renovateTag": "RPM_registry=https://packages.microsoft.com/azurelinux/3.0/prod/cloud-native/x86_64/repodata, name=node-exporter-kubernetes, os=azurelinux, release=3.0",
-                "latestVersion": "1.9.1-28.azl3"
+                "latestVersion": "1.12.1-7.azl3"
               }
             ]
           },