chore(deps): update node exporter to 1.12.1

e2e/nodeexporter/metrics.go

@@ -2,23 +2,20 @@ package nodeexporter
 
 import (
 	"fmt"
+	"regexp"
+	"strconv"
 	"strings"
 )
 
+var (
+	metricNamePattern          = regexp.MustCompile(`^([^{ \t]+)(?:\{[^}]*\})?[ \t]+`)
+	nodeExporterVersionPattern = regexp.MustCompile(`(?m)^node_exporter_build_info\{[^\n}]*version="v([0-9]+)\.([0-9]+)\.[^"}]+"[^\n}]*\} 1$`)
+)
+
 // ValidateMetrics verifies that a node-exporter scrape contains every metric
 // retained by the AKS default Prometheus profile.
 func ValidateMetrics(metricsText string) error {
-	metricNames := make(map[string]struct{})
-	for line := range strings.SplitSeq(metricsText, "\n") {
-		line = strings.TrimSpace(line)
-		if line == "" || strings.HasPrefix(line, "#") {
-			continue
-		}
-
-		if end := strings.IndexAny(line, "{ \t"); end > 0 {
-			metricNames[line[:end]] = struct{}{}
-		}
-	}
+	metricNames := parseMetricNames(metricsText)
 
 	requiredMetrics := []string{
 		"node_disk_read_time_seconds_total",
@@ -49,3 +46,93 @@ func ValidateMetrics(metricsText string) error {
 
 	return nil
 }
+
+func parseMetricNames(metricsText string) map[string]struct{} {
+	metricNames := make(map[string]struct{})
+	for line := range strings.SplitSeq(metricsText, "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+
+		if match := metricNamePattern.FindStringSubmatch(line); len(match) == 2 {
+			metricNames[match[1]] = struct{}{}
+		}
+	}
+	return metricNames
+}
+
+// ValidateCollectors verifies that the collectors enabled by AgentBaker are present in the scrape.
+// InfiniBand metrics are required only when the node has InfiniBand hardware.
+func ValidateCollectors(metricsText string, requireInfiniBand bool) error {
+	isVersion112OrNewer, err := nodeExporterVersionAtLeast(metricsText, 1, 12)
+	if err != nil {
+		return err
+	}
+	if isVersion112OrNewer {
+		requiredCollectors := []string{"bcachefs", "dmmultipath", "kernel_hung"}
+		var missingCollectors []string
+		for _, collector := range requiredCollectors {
+			if !strings.Contains(metricsText, `node_scrape_collector_success{collector="`+collector+`"}`) {
+				missingCollectors = append(missingCollectors, collector)
+			}
+		}
+		if len(missingCollectors) > 0 {
+			return fmt.Errorf("enabled collectors are missing: %s", strings.Join(missingCollectors, ", "))
+		}
+	}
+
+	if requireInfiniBand {
+		if !strings.Contains(metricsText, `node_scrape_collector_success{collector="infiniband"} 1`) {
+			return fmt.Errorf("InfiniBand collector did not succeed")
+		}
+		hasInfiniBandMetric := false
+		for name := range parseMetricNames(metricsText) {
+			if strings.HasPrefix(name, "node_infiniband_") {
+				hasInfiniBandMetric = true
+				break
+			}
+		}
+		if !hasInfiniBandMetric {
+			return fmt.Errorf("InfiniBand metrics are missing")
+		}
+
+		if isVersion112OrNewer {
+			hardwareCounterMetrics := []string{
+				"node_infiniband_duplicate_requests_packets_total",
+				"node_infiniband_lifespan_seconds",
+				"node_infiniband_out_of_buffer_drops_total",
+				"node_infiniband_rx_write_requests_total",
+			}
+			hasHardwareCounterMetric := false
+			metricNames := parseMetricNames(metricsText)
+			for _, name := range hardwareCounterMetrics {
+				if _, exists := metricNames[name]; exists {
+					hasHardwareCounterMetric = true
+					break
+				}
+			}
+			if !hasHardwareCounterMetric {
+				return fmt.Errorf("InfiniBand hardware counter metrics are missing")
+			}
+		}
+	}
+
+	return nil
+}
+
+func nodeExporterVersionAtLeast(metricsText string, requiredMajor, requiredMinor int) (bool, error) {
+	match := nodeExporterVersionPattern.FindStringSubmatch(metricsText)
+	if len(match) != 3 {
+		return false, fmt.Errorf("node exporter build info is missing a valid version")
+	}
+	major, err := strconv.Atoi(match[1])
+	if err != nil {
+		return false, fmt.Errorf("parse node exporter major version: %w", err)
+	}
+	minor, err := strconv.Atoi(match[2])
+	if err != nil {
+		return false, fmt.Errorf("parse node exporter minor version: %w", err)
+	}
+	return major > requiredMajor || major == requiredMajor && minor >= requiredMinor, nil
+}

e2e/nodeexporter/metrics_test.go

@@ -23,6 +23,57 @@ func TestValidateMetricsRejectsMissingMetric(t *testing.T) {
 	require.ErrorContains(t, err, "node_network_receive_bytes_total")
 }
 
+func TestValidateCollectors(t *testing.T) {
+	metrics := `node_exporter_build_info{version="v1.12.1"} 1
+node_scrape_collector_success{collector="bcachefs"} 0
+node_scrape_collector_success{collector="dmmultipath"} 1
+node_scrape_collector_success{collector="kernel_hung"} 0
+`
+
+	require.NoError(t, ValidateCollectors(metrics, false))
+}
+
+func TestValidateCollectorsRejectsMissingCollector(t *testing.T) {
+	metrics := `node_exporter_build_info{version="v1.12.1"} 1
+node_scrape_collector_success{collector="bcachefs"} 0
+node_scrape_collector_success{collector="kernel_hung"} 0
+`
+
+	require.ErrorContains(t, ValidateCollectors(metrics, false), "dmmultipath")
+}
+
+func TestValidateCollectorsRequiresInfiniBandMetricsOnInfiniBandNodes(t *testing.T) {
+	metrics := `node_exporter_build_info{version="v1.12.1"} 1
+node_scrape_collector_success{collector="bcachefs"} 0
+node_scrape_collector_success{collector="dmmultipath"} 1
+node_scrape_collector_success{collector="kernel_hung"} 0
+node_scrape_collector_success{collector="infiniband"} 1
+node_infiniband_port_state{device="mlx5_0",port="1",state="ACTIVE"} 1
+node_infiniband_out_of_buffer_drops_total{device="mlx5_0",port="1"} 0
+`
+
+	require.NoError(t, ValidateCollectors(metrics, true))
+	require.ErrorContains(t, ValidateCollectors(strings.Replace(metrics, "node_infiniband_", "other_metric_", -1), true), "metrics are missing")
+	require.ErrorContains(t, ValidateCollectors(strings.Replace(metrics, "\nnode_infiniband_out_of_buffer_drops_total", "\nother_metric", 1), true), "hardware counter metrics are missing")
+	require.ErrorContains(t, ValidateCollectors(strings.Replace(metrics, `collector="infiniband"} 1`, `collector="infiniband"} 0`, 1), true), "did not succeed")
+}
+
+func TestValidateCollectorsSupportsMainVHDExporter(t *testing.T) {
+	metrics := `node_exporter_build_info{version="v1.9.1"} 1`
+
+	require.NoError(t, ValidateCollectors(metrics, false))
+}
+
+func TestValidateCollectorsRequiresNewCollectorsAfter112(t *testing.T) {
+	metrics := `node_exporter_build_info{version="v1.13.0"} 1`
+
+	require.ErrorContains(t, ValidateCollectors(metrics, false), "bcachefs")
+}
+
+func TestValidateCollectorsRejectsMissingBuildInfo(t *testing.T) {
+	require.ErrorContains(t, ValidateCollectors("", false), "build info")
+}
+
 func validMetrics() string {
 	return `# TYPE node_disk_read_time_seconds_total counter
 node_disk_read_time_seconds_total{device="sda"} 1

e2e/validators.go

@@ -2827,7 +2827,11 @@ func ValidateNodeExporter(ctx context.Context, s *Scenario) error {
 	// so this also verifies that the endpoint is reachable on the address used by monitoring infrastructure.
 	s.Logger.Logf("Validating node-exporter metrics on port 19100")
 	metricsURL := fmt.Sprintf("http://%s:19100/metrics", s.Runtime.VM.PrivateIP)
-	errs = append(errs, scrapeAndValidateNodeExporter(ctx, s, metricsURL))
+	hasInfiniBand, err := nodeHasInfiniBandHardware(ctx, s)
+	if err != nil {
+		errs = append(errs, err)
+	}
+	errs = append(errs, scrapeAndValidateNodeExporter(ctx, s, metricsURL, hasInfiniBand))
 
 	if _, err := execScriptOnVMForScenarioValidateExitCode(ctx, s, fmt.Sprintf("systemctl is-active %s", serviceName), 0,
 		"node-exporter should remain active after scraping"); err != nil {
@@ -2841,7 +2845,31 @@ func ValidateNodeExporter(ctx context.Context, s *Scenario) error {
 	return nil
 }
 
-func scrapeAndValidateNodeExporter(ctx context.Context, s *Scenario, metricsURL string) error {
+func nodeHasInfiniBandHardware(ctx context.Context, s *Scenario) (bool, error) {
+	// node-exporter 1.12.1 cannot parse MANA RDMA devices, so the startup script
+	// disables the collector when MANA PCI hardware is present. Keep this
+	// detection aligned so MANA and mixed-HCA nodes do not require metrics from a
+	// collector that must be disabled pending an upstream fix.
+	command := `for device in /sys/bus/pci/devices/*; do
+    if [ -d "$device" ] &&
+       grep -qi '^0x1414$' "$device/vendor" 2>/dev/null &&
+       grep -Eqi '^0x00(b9|ba|c1)$' "$device/device" 2>/dev/null; then
+        exit 1
+    fi
+done
+for device in /sys/class/infiniband/*; do
+    [ -e "$device" ] || continue
+    exit 0
+done
+exit 1`
+	result, err := execScriptOnVMForScenario(ctx, s, command)
+	if err != nil {
+		return false, fmt.Errorf("detect InfiniBand hardware: %w", err)
+	}
+	return result.exitCode == "0", nil
+}
+
+func scrapeAndValidateNodeExporter(ctx context.Context, s *Scenario, metricsURL string, requireInfiniBand bool) error {
 	result, err := execScriptOnVMForScenario(ctx, s, fmt.Sprintf("curl --noproxy '*' -sS --max-time 10 %q", metricsURL))
 	if err != nil {
 		return fmt.Errorf("scrape node-exporter metrics from %s: %w", metricsURL, err)
@@ -2856,7 +2884,10 @@ func scrapeAndValidateNodeExporter(ctx context.Context, s *Scenario, metricsURL
 	if len(responsePreview) > previewLimit {
 		responsePreview = responsePreview[:previewLimit] + "\n... response truncated"
 	}
-	return assert.NoError(nodeexporter.ValidateMetrics(result.stdout), "node-exporter scrape did not satisfy the AKS Prometheus metrics contract\nresponse preview:\n%s", responsePreview)
+	return errors.Join(
+		assert.NoError(nodeexporter.ValidateMetrics(result.stdout), "node-exporter scrape did not satisfy the AKS Prometheus metrics contract\nresponse preview:\n%s", responsePreview),
+		assert.NoError(nodeexporter.ValidateCollectors(result.stdout, requireInfiniBand), "node-exporter collectors did not satisfy the AgentBaker contract\nresponse preview:\n%s", responsePreview),
+	)
 }
 
 func ValidateNPDFilesystemCorruption(ctx context.Context, s *Scenario) (err error) {

parts/common/components.json

@@ -2183,31 +2183,31 @@
             "versionsV2": [
               {
                 "renovateTag": "name=node-exporter-kubernetes, repository=production, os=ubuntu, release=26.04",
-                "latestVersion": "1.9.1-ubuntu26.04u28"
+                "latestVersion": "1.12.1-ubuntu26.04u7"
               }
             ]
           },
           "r2404": {
             "versionsV2": [
               {
                 "renovateTag": "name=node-exporter-kubernetes, repository=production, os=ubuntu, release=24.04",
-                "latestVersion": "1.9.1-ubuntu24.04u28"
+                "latestVersion": "1.12.1-ubuntu24.04u7"
               }
             ]
           },
           "r2204": {
             "versionsV2": [
               {
                 "renovateTag": "name=node-exporter-kubernetes, repository=production, os=ubuntu, release=22.04",
-                "latestVersion": "1.9.1-ubuntu22.04u28"
+                "latestVersion": "1.12.1-ubuntu22.04u7"
               }
             ]
           },
           "r2004": {
             "versionsV2": [
               {
                 "renovateTag": "name=node-exporter-kubernetes, repository=production, os=ubuntu, release=20.04",
-                "latestVersion": "1.9.1-ubuntu20.04u28"
+                "latestVersion": "1.12.1-ubuntu20.04u7"
               }
             ]
           }
@@ -2217,7 +2217,7 @@
             "versionsV2": [
               {
                 "renovateTag": "RPM_registry=https://packages.microsoft.com/azurelinux/3.0/prod/cloud-native/x86_64/repodata, name=node-exporter-kubernetes, os=azurelinux, release=3.0",
-                "latestVersion": "1.9.1-28.azl3"
+                "latestVersion": "1.12.1-7.azl3"
               }
             ]
           },

parts/linux/cloud-init/artifacts/node-exporter/node-exporter-startup.sh

@@ -1,5 +1,22 @@
 #!/bin/bash
 
+PCI_DEVICES_PATH="${PCI_DEVICES_PATH:-/sys/bus/pci/devices}"
+
+getNodeExporterHardwareArgs() {
+    for device in "${PCI_DEVICES_PATH}"/*; do
+        if [ -d "$device" ] &&
+           grep -qi '^0x1414$' "$device/vendor" 2>/dev/null &&
+           grep -Eqi '^0x00(b9|ba|c1)$' "$device/device" 2>/dev/null; then
+            printf '%s\n' '--no-collector.infiniband'
+            return
+        fi
+    done
+}
+
+if [ "${NODE_EXPORTER_STARTUP_SOURCE_ONLY:-false}" = "true" ]; then
+    return 0
+fi
+
 if [ "$(grep ^ID= /etc/os-release | cut -c 4-)" = "flatcar" ]; then
     NODE_IP=$(ip -o -4 addr show dev eth0 | awk '{print $4}' | cut -d '/' -f 1)
 else
@@ -99,6 +116,22 @@ ARGS=(
     --no-collector.arp.netlink
 )
 
+# MANA's RDMA driver publicly supports /sys/class/infiniband, but its rate file
+# returns EINVAL with the parser used by node-exporter 1.12.1. node-exporter also
+# parses every device before applying either its device include or exclude
+# filter, so neither flag can avoid the failure. Detect MANA by its assigned PCI
+# IDs (Microsoft 1414; MANA PF 00b9, VF 00ba, PF2 00c1). PCI enumeration happens
+# during boot before userspace services start, unlike the later mana_ib driver
+# registration that creates mana_* under /sys/class/infiniband, so this check
+# cannot race RDMA class creation. Disable the whole collector when MANA is
+# present. This also suppresses metrics from other HCAs on mixed-HCA nodes until
+# upstream can filter before parsing devices.
+# https://github.com/prometheus/node_exporter/issues/3810
+# https://learn.microsoft.com/azure/virtual-network/accelerated-networking-mana-linux
+# https://github.com/torvalds/linux/blob/master/include/net/mana/gdma.h
+readarray -t HARDWARE_ARGS < <(getNodeExporterHardwareArgs)
+ARGS+=("${HARDWARE_ARGS[@]}")
+
 if [ -n "$TLS_CONFIG_ARG" ]; then
     ARGS+=("$TLS_CONFIG_ARG")
 fi

spec/parts/linux/cloud-init/artifacts/node-exporter-startup_spec.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+
+Describe 'node-exporter-startup.sh hardware arguments'
+    NODE_EXPORTER_STARTUP_SOURCE_ONLY=true
+    Include './parts/linux/cloud-init/artifacts/node-exporter/node-exporter-startup.sh'
+
+    setup_pci_devices() {
+        PCI_DEVICES_PATH="$(mktemp -d)"
+    }
+
+    add_pci_device() {
+        local name="$1"
+        local vendor="$2"
+        local device="$3"
+        mkdir "${PCI_DEVICES_PATH}/${name}"
+        printf '%s\n' "$vendor" > "${PCI_DEVICES_PATH}/${name}/vendor"
+        printf '%s\n' "$device" > "${PCI_DEVICES_PATH}/${name}/device"
+    }
+
+    BeforeEach 'setup_pci_devices'
+    AfterEach 'rm -rf "$PCI_DEVICES_PATH"'
+
+    It 'adds no argument without PCI devices'
+        When call getNodeExporterHardwareArgs
+        The status should be success
+        The output should equal ''
+    End
+
+    It 'keeps the InfiniBand collector enabled for non-MANA PCI devices'
+        add_pci_device '0000:00:02.0' '0x15b3' '0x1017'
+
+        When call getNodeExporterHardwareArgs
+        The status should be success
+        The output should equal ''
+    End
+
+    It 'disables the InfiniBand collector for MANA devices'
+        add_pci_device '7870:00:00.0' '0x1414' '0x00ba'
+
+        When call getNodeExporterHardwareArgs
+        The status should be success
+        The output should equal '--no-collector.infiniband'
+    End
+
+    It 'recognizes the current MANA PF and PF2 device IDs'
+        add_pci_device '7870:00:00.0' '0x1414' '0x00b9'
+        add_pci_device '7870:00:00.1' '0x1414' '0x00c1'
+
+        When call getNodeExporterHardwareArgs
+        The status should be success
+        The output should equal '--no-collector.infiniband'
+    End
+
+    It 'disables the InfiniBand collector on mixed MANA and non-MANA devices'
+        add_pci_device '0000:00:02.0' '0x15b3' '0x1017'
+        add_pci_device '7870:00:00.0' '0x1414' '0x00ba'
+
+        When call getNodeExporterHardwareArgs
+        The status should be success
+        The output should equal '--no-collector.infiniband'
+    End
+End