From 5155a03a0b2879972a2f5ce6e9b5557aae956098 Mon Sep 17 00:00:00 2001
From: chmill <chmill@microsoft.com>
Date: Fri, 4 Sep 2026 02:27:15 +0000
Subject: [PATCH 1/7] feat: update node exporter to 1.12.1

---
 e2e/nodeexporter/metrics.go                   | 109 ++++++++++++++++--
 e2e/nodeexporter/metrics_test.go              |  51 ++++++++
 e2e/validators.go                             |  22 +++-
 parts/common/components.json                  |  10 +-
 .../packer/test/linux-vhd-content-test.sh     |  16 +++
 5 files changed, 189 insertions(+), 19 deletions(-)

diff --git a/e2e/nodeexporter/metrics.go b/e2e/nodeexporter/metrics.go
index 380026f2d61..5ec6c13fc16 100644
--- a/e2e/nodeexporter/metrics.go
+++ b/e2e/nodeexporter/metrics.go
@@ -2,23 +2,20 @@ package nodeexporter
 
 import (
 	"fmt"
+	"regexp"
+	"strconv"
 	"strings"
 )
 
+var (
+	metricNamePattern          = regexp.MustCompile(`^([^{ \t]+)(?:\{[^}]*\})?[ \t]+`)
+	nodeExporterVersionPattern = regexp.MustCompile(`(?m)^node_exporter_build_info\{[^\n}]*version="v([0-9]+)\.([0-9]+)\.[^"}]+"[^\n}]*\} 1$`)
+)
+
 // ValidateMetrics verifies that a node-exporter scrape contains every metric
 // retained by the AKS default Prometheus profile.
 func ValidateMetrics(metricsText string) error {
-	metricNames := make(map[string]struct{})
-	for line := range strings.SplitSeq(metricsText, "\n") {
-		line = strings.TrimSpace(line)
-		if line == "" || strings.HasPrefix(line, "#") {
-			continue
-		}
-
-		if end := strings.IndexAny(line, "{ \t"); end > 0 {
-			metricNames[line[:end]] = struct{}{}
-		}
-	}
+	metricNames := parseMetricNames(metricsText)
 
 	requiredMetrics := []string{
 		"node_disk_read_time_seconds_total",
@@ -49,3 +46,93 @@ func ValidateMetrics(metricsText string) error {
 
 	return nil
 }
+
+func parseMetricNames(metricsText string) map[string]struct{} {
+	metricNames := make(map[string]struct{})
+	for line := range strings.SplitSeq(metricsText, "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+
+		if match := metricNamePattern.FindStringSubmatch(line); len(match) == 2 {
+			metricNames[match[1]] = struct{}{}
+		}
+	}
+	return metricNames
+}
+
+// ValidateCollectors verifies that the collectors enabled by AgentBaker are present in the scrape.
+// InfiniBand metrics are required only when the node has InfiniBand hardware.
+func ValidateCollectors(metricsText string, requireInfiniBand bool) error {
+	isVersion112OrNewer, err := nodeExporterVersionAtLeast(metricsText, 1, 12)
+	if err != nil {
+		return err
+	}
+	if isVersion112OrNewer {
+		requiredCollectors := []string{"bcachefs", "dmmultipath", "kernel_hung"}
+		var missingCollectors []string
+		for _, collector := range requiredCollectors {
+			if !strings.Contains(metricsText, `node_scrape_collector_success{collector="`+collector+`"}`) {
+				missingCollectors = append(missingCollectors, collector)
+			}
+		}
+		if len(missingCollectors) > 0 {
+			return fmt.Errorf("enabled collectors are missing: %s", strings.Join(missingCollectors, ", "))
+		}
+	}
+
+	if requireInfiniBand {
+		if !strings.Contains(metricsText, `node_scrape_collector_success{collector="infiniband"} 1`) {
+			return fmt.Errorf("InfiniBand collector did not succeed")
+		}
+		hasInfiniBandMetric := false
+		for name := range parseMetricNames(metricsText) {
+			if strings.HasPrefix(name, "node_infiniband_") {
+				hasInfiniBandMetric = true
+				break
+			}
+		}
+		if !hasInfiniBandMetric {
+			return fmt.Errorf("InfiniBand metrics are missing")
+		}
+
+		if isVersion112OrNewer {
+			hardwareCounterMetrics := []string{
+				"node_infiniband_duplicate_requests_packets_total",
+				"node_infiniband_lifespan_seconds",
+				"node_infiniband_out_of_buffer_drops_total",
+				"node_infiniband_rx_write_requests_total",
+			}
+			hasHardwareCounterMetric := false
+			metricNames := parseMetricNames(metricsText)
+			for _, name := range hardwareCounterMetrics {
+				if _, exists := metricNames[name]; exists {
+					hasHardwareCounterMetric = true
+					break
+				}
+			}
+			if !hasHardwareCounterMetric {
+				return fmt.Errorf("InfiniBand hardware counter metrics are missing")
+			}
+		}
+	}
+
+	return nil
+}
+
+func nodeExporterVersionAtLeast(metricsText string, requiredMajor, requiredMinor int) (bool, error) {
+	match := nodeExporterVersionPattern.FindStringSubmatch(metricsText)
+	if len(match) != 3 {
+		return false, fmt.Errorf("node exporter build info is missing a valid version")
+	}
+	major, err := strconv.Atoi(match[1])
+	if err != nil {
+		return false, fmt.Errorf("parse node exporter major version: %w", err)
+	}
+	minor, err := strconv.Atoi(match[2])
+	if err != nil {
+		return false, fmt.Errorf("parse node exporter minor version: %w", err)
+	}
+	return major > requiredMajor || major == requiredMajor && minor >= requiredMinor, nil
+}
diff --git a/e2e/nodeexporter/metrics_test.go b/e2e/nodeexporter/metrics_test.go
index 4ddecd2c5f2..3f4bc1a2520 100644
--- a/e2e/nodeexporter/metrics_test.go
+++ b/e2e/nodeexporter/metrics_test.go
@@ -23,6 +23,57 @@ func TestValidateMetricsRejectsMissingMetric(t *testing.T) {
 	require.ErrorContains(t, err, "node_network_receive_bytes_total")
 }
 
+func TestValidateCollectors(t *testing.T) {
+	metrics := `node_exporter_build_info{version="v1.12.1"} 1
+node_scrape_collector_success{collector="bcachefs"} 0
+node_scrape_collector_success{collector="dmmultipath"} 1
+node_scrape_collector_success{collector="kernel_hung"} 0
+`
+
+	require.NoError(t, ValidateCollectors(metrics, false))
+}
+
+func TestValidateCollectorsRejectsMissingCollector(t *testing.T) {
+	metrics := `node_exporter_build_info{version="v1.12.1"} 1
+node_scrape_collector_success{collector="bcachefs"} 0
+node_scrape_collector_success{collector="kernel_hung"} 0
+`
+
+	require.ErrorContains(t, ValidateCollectors(metrics, false), "dmmultipath")
+}
+
+func TestValidateCollectorsRequiresInfiniBandMetricsOnInfiniBandNodes(t *testing.T) {
+	metrics := `node_exporter_build_info{version="v1.12.1"} 1
+node_scrape_collector_success{collector="bcachefs"} 0
+node_scrape_collector_success{collector="dmmultipath"} 1
+node_scrape_collector_success{collector="kernel_hung"} 0
+node_scrape_collector_success{collector="infiniband"} 1
+node_infiniband_port_state{device="mlx5_0",port="1",state="ACTIVE"} 1
+node_infiniband_out_of_buffer_drops_total{device="mlx5_0",port="1"} 0
+`
+
+	require.NoError(t, ValidateCollectors(metrics, true))
+	require.ErrorContains(t, ValidateCollectors(strings.Replace(metrics, "node_infiniband_", "other_metric_", -1), true), "metrics are missing")
+	require.ErrorContains(t, ValidateCollectors(strings.Replace(metrics, "\nnode_infiniband_out_of_buffer_drops_total", "\nother_metric", 1), true), "hardware counter metrics are missing")
+	require.ErrorContains(t, ValidateCollectors(strings.Replace(metrics, `collector="infiniband"} 1`, `collector="infiniband"} 0`, 1), true), "did not succeed")
+}
+
+func TestValidateCollectorsSupportsMainVHDExporter(t *testing.T) {
+	metrics := `node_exporter_build_info{version="v1.9.1"} 1`
+
+	require.NoError(t, ValidateCollectors(metrics, false))
+}
+
+func TestValidateCollectorsRequiresNewCollectorsAfter112(t *testing.T) {
+	metrics := `node_exporter_build_info{version="v1.13.0"} 1`
+
+	require.ErrorContains(t, ValidateCollectors(metrics, false), "bcachefs")
+}
+
+func TestValidateCollectorsRejectsMissingBuildInfo(t *testing.T) {
+	require.ErrorContains(t, ValidateCollectors("", false), "build info")
+}
+
 func validMetrics() string {
 	return `# TYPE node_disk_read_time_seconds_total counter
 node_disk_read_time_seconds_total{device="sda"} 1
diff --git a/e2e/validators.go b/e2e/validators.go
index b6562feb8e8..57985575dd5 100644
--- a/e2e/validators.go
+++ b/e2e/validators.go
@@ -2827,7 +2827,11 @@ func ValidateNodeExporter(ctx context.Context, s *Scenario) error {
 	// so this also verifies that the endpoint is reachable on the address used by monitoring infrastructure.
 	s.Logger.Logf("Validating node-exporter metrics on port 19100")
 	metricsURL := fmt.Sprintf("http://%s:19100/metrics", s.Runtime.VM.PrivateIP)
-	errs = append(errs, scrapeAndValidateNodeExporter(ctx, s, metricsURL))
+	hasInfiniBand, err := nodeHasInfiniBandHardware(ctx, s)
+	if err != nil {
+		errs = append(errs, err)
+	}
+	errs = append(errs, scrapeAndValidateNodeExporter(ctx, s, metricsURL, hasInfiniBand))
 
 	if _, err := execScriptOnVMForScenarioValidateExitCode(ctx, s, fmt.Sprintf("systemctl is-active %s", serviceName), 0,
 		"node-exporter should remain active after scraping"); err != nil {
@@ -2841,7 +2845,16 @@ func ValidateNodeExporter(ctx context.Context, s *Scenario) error {
 	return nil
 }
 
-func scrapeAndValidateNodeExporter(ctx context.Context, s *Scenario, metricsURL string) error {
+func nodeHasInfiniBandHardware(ctx context.Context, s *Scenario) (bool, error) {
+	command := `for device in /sys/class/infiniband/*; do [ -e "$device" ] && exit 0; done; exit 1`
+	result, err := execScriptOnVMForScenario(ctx, s, command)
+	if err != nil {
+		return false, fmt.Errorf("detect InfiniBand hardware: %w", err)
+	}
+	return result.exitCode == "0", nil
+}
+
+func scrapeAndValidateNodeExporter(ctx context.Context, s *Scenario, metricsURL string, requireInfiniBand bool) error {
 	result, err := execScriptOnVMForScenario(ctx, s, fmt.Sprintf("curl --noproxy '*' -sS --max-time 10 %q", metricsURL))
 	if err != nil {
 		return fmt.Errorf("scrape node-exporter metrics from %s: %w", metricsURL, err)
@@ -2856,7 +2869,10 @@ func scrapeAndValidateNodeExporter(ctx context.Context, s *Scenario, metricsURL
 	if len(responsePreview) > previewLimit {
 		responsePreview = responsePreview[:previewLimit] + "\n... response truncated"
 	}
-	return assert.NoError(nodeexporter.ValidateMetrics(result.stdout), "node-exporter scrape did not satisfy the AKS Prometheus metrics contract\nresponse preview:\n%s", responsePreview)
+	return errors.Join(
+		assert.NoError(nodeexporter.ValidateMetrics(result.stdout), "node-exporter scrape did not satisfy the AKS Prometheus metrics contract\nresponse preview:\n%s", responsePreview),
+		assert.NoError(nodeexporter.ValidateCollectors(result.stdout, requireInfiniBand), "node-exporter collectors did not satisfy the AgentBaker contract\nresponse preview:\n%s", responsePreview),
+	)
 }
 
 func ValidateNPDFilesystemCorruption(ctx context.Context, s *Scenario) (err error) {
diff --git a/parts/common/components.json b/parts/common/components.json
index c350ff7e54c..3baccf83f90 100644
--- a/parts/common/components.json
+++ b/parts/common/components.json
@@ -2183,7 +2183,7 @@
             "versionsV2": [
               {
                 "renovateTag": "name=node-exporter-kubernetes, repository=production, os=ubuntu, release=26.04",
-                "latestVersion": "1.9.1-ubuntu26.04u28"
+                "latestVersion": "1.12.1-ubuntu26.04u7"
               }
             ]
           },
@@ -2191,7 +2191,7 @@
             "versionsV2": [
               {
                 "renovateTag": "name=node-exporter-kubernetes, repository=production, os=ubuntu, release=24.04",
-                "latestVersion": "1.9.1-ubuntu24.04u28"
+                "latestVersion": "1.12.1-ubuntu24.04u7"
               }
             ]
           },
@@ -2199,7 +2199,7 @@
             "versionsV2": [
               {
                 "renovateTag": "name=node-exporter-kubernetes, repository=production, os=ubuntu, release=22.04",
-                "latestVersion": "1.9.1-ubuntu22.04u28"
+                "latestVersion": "1.12.1-ubuntu22.04u7"
               }
             ]
           },
@@ -2207,7 +2207,7 @@
             "versionsV2": [
               {
                 "renovateTag": "name=node-exporter-kubernetes, repository=production, os=ubuntu, release=20.04",
-                "latestVersion": "1.9.1-ubuntu20.04u28"
+                "latestVersion": "1.12.1-ubuntu20.04u7"
               }
             ]
           }
@@ -2217,7 +2217,7 @@
             "versionsV2": [
               {
                 "renovateTag": "RPM_registry=https://packages.microsoft.com/azurelinux/3.0/prod/cloud-native/x86_64/repodata, name=node-exporter-kubernetes, os=azurelinux, release=3.0",
-                "latestVersion": "1.9.1-28.azl3"
+                "latestVersion": "1.12.1-7.azl3"
               }
             ]
           },
diff --git a/vhdbuilder/packer/test/linux-vhd-content-test.sh b/vhdbuilder/packer/test/linux-vhd-content-test.sh
index 1ec1342d623..1c0d214080a 100644
--- a/vhdbuilder/packer/test/linux-vhd-content-test.sh
+++ b/vhdbuilder/packer/test/linux-vhd-content-test.sh
@@ -1993,6 +1993,22 @@ testNodeExporter () {
   fi
   echo "$test: skip sentinel file exists at $skip_file"
 
+  local expectedVersion
+  expectedVersion=$(getPackageExpectedVersion "node-exporter" "" "" "")
+  if [ "$expectedVersion" = "<SKIP>" ]; then
+    err "$test" "node-exporter expected version is <SKIP> on supported OS $os_sku"
+    return 1
+  fi
+  assertPackageVersion "$test" "node-exporter-kubernetes" "$expectedVersion" || return 1
+
+  local expectedBinaryVersion="v${expectedVersion%%-*}"
+  local binaryVersion
+  binaryVersion=$(/usr/bin/node-exporter --version 2>&1 | awk 'NR == 1 { print $3 }')
+  if [ "$binaryVersion" != "$expectedBinaryVersion" ]; then
+    err "$test" "node-exporter binary version '$binaryVersion' does not match expected '$expectedBinaryVersion'"
+    return 1
+  fi
+
   # The Dalec-built deb/rpm installs the binary to /usr/bin/node-exporter.
   # We then create a symlink at /opt/bin/node-exporter for consistency with
   # other binaries (kubelet, kubectl) that live in /opt/bin.

From 8ce5075ca91b3e5ccfe2764a10179bbc6364abb5 Mon Sep 17 00:00:00 2001
From: chmill <chmill@microsoft.com>
Date: Fri, 4 Sep 2026 19:07:43 +0000
Subject: [PATCH 2/7] fix: handle MANA node exporter collection

---
 e2e/validators.go                                 |  9 ++++++++-
 .../node-exporter/node-exporter-startup.sh        | 15 +++++++++++++++
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/e2e/validators.go b/e2e/validators.go
index 57985575dd5..a66b7dc47e5 100644
--- a/e2e/validators.go
+++ b/e2e/validators.go
@@ -2846,7 +2846,14 @@ func ValidateNodeExporter(ctx context.Context, s *Scenario) error {
 }
 
 func nodeHasInfiniBandHardware(ctx context.Context, s *Scenario) (bool, error) {
-	command := `for device in /sys/class/infiniband/*; do [ -e "$device" ] && exit 0; done; exit 1`
+	command := `for device in /sys/class/infiniband/mana_*; do
+    [ -e "$device" ] && exit 1
+done
+for device in /sys/class/infiniband/*; do
+    [ -e "$device" ] || continue
+    exit 0
+done
+exit 1`
 	result, err := execScriptOnVMForScenario(ctx, s, command)
 	if err != nil {
 		return false, fmt.Errorf("detect InfiniBand hardware: %w", err)
diff --git a/parts/linux/cloud-init/artifacts/node-exporter/node-exporter-startup.sh b/parts/linux/cloud-init/artifacts/node-exporter/node-exporter-startup.sh
index 37299a6ca8e..8285b45f755 100755
--- a/parts/linux/cloud-init/artifacts/node-exporter/node-exporter-startup.sh
+++ b/parts/linux/cloud-init/artifacts/node-exporter/node-exporter-startup.sh
@@ -99,6 +99,21 @@ ARGS=(
     --no-collector.arp.netlink
 )
 
+# MANA registers RDMA devices under /sys/class/infiniband, but node-exporter
+# fails the entire InfiniBand collector while parsing their rate files. Device
+# filters are applied too late to avoid the failure, so disable the collector
+# whenever MANA is present, including on mixed-HCA nodes.
+HAS_MANA_RDMA_DEVICE=false
+for device in /sys/class/infiniband/*; do
+    [ -e "$device" ] || continue
+    case "$(basename "$device")" in
+        mana_*) HAS_MANA_RDMA_DEVICE=true ;;
+    esac
+done
+if [ "$HAS_MANA_RDMA_DEVICE" = "true" ]; then
+    ARGS+=(--no-collector.infiniband)
+fi
+
 if [ -n "$TLS_CONFIG_ARG" ]; then
     ARGS+=("$TLS_CONFIG_ARG")
 fi

From 59cd2e05a0deb1f63593f3afed07afdcfaf35c3a Mon Sep 17 00:00:00 2001
From: chmill <chmill@microsoft.com>
Date: Fri, 4 Sep 2026 19:23:03 +0000
Subject: [PATCH 3/7] test: cover MANA node exporter arguments

---
 e2e/validators.go                             |  4 ++
 .../node-exporter/node-exporter-startup.sh    | 39 +++++++++++------
 .../artifacts/node-exporter-startup_spec.sh   | 43 +++++++++++++++++++
 3 files changed, 72 insertions(+), 14 deletions(-)
 create mode 100644 spec/parts/linux/cloud-init/artifacts/node-exporter-startup_spec.sh

diff --git a/e2e/validators.go b/e2e/validators.go
index a66b7dc47e5..5e5ac4cc62d 100644
--- a/e2e/validators.go
+++ b/e2e/validators.go
@@ -2846,6 +2846,10 @@ func ValidateNodeExporter(ctx context.Context, s *Scenario) error {
 }
 
 func nodeHasInfiniBandHardware(ctx context.Context, s *Scenario) (bool, error) {
+	// MANA is an RDMA device under /sys/class/infiniband, but node-exporter 1.12.1
+	// cannot parse it and the startup script disables the collector when present.
+	// Keep this detection aligned so MANA and mixed-HCA nodes do not require
+	// metrics from a collector that must be disabled pending an upstream fix.
 	command := `for device in /sys/class/infiniband/mana_*; do
     [ -e "$device" ] && exit 1
 done
diff --git a/parts/linux/cloud-init/artifacts/node-exporter/node-exporter-startup.sh b/parts/linux/cloud-init/artifacts/node-exporter/node-exporter-startup.sh
index 8285b45f755..75193cec037 100755
--- a/parts/linux/cloud-init/artifacts/node-exporter/node-exporter-startup.sh
+++ b/parts/linux/cloud-init/artifacts/node-exporter/node-exporter-startup.sh
@@ -1,5 +1,20 @@
 #!/bin/bash
 
+INFINIBAND_CLASS_PATH="${INFINIBAND_CLASS_PATH:-/sys/class/infiniband}"
+
+getNodeExporterHardwareArgs() {
+    for device in "${INFINIBAND_CLASS_PATH}"/mana_*; do
+        if [ -e "$device" ]; then
+            printf '%s\n' '--no-collector.infiniband'
+            return
+        fi
+    done
+}
+
+if [ "${NODE_EXPORTER_STARTUP_SOURCE_ONLY:-false}" = "true" ]; then
+    return 0
+fi
+
 if [ "$(grep ^ID= /etc/os-release | cut -c 4-)" = "flatcar" ]; then
     NODE_IP=$(ip -o -4 addr show dev eth0 | awk '{print $4}' | cut -d '/' -f 1)
 else
@@ -99,20 +114,16 @@ ARGS=(
     --no-collector.arp.netlink
 )
 
-# MANA registers RDMA devices under /sys/class/infiniband, but node-exporter
-# fails the entire InfiniBand collector while parsing their rate files. Device
-# filters are applied too late to avoid the failure, so disable the collector
-# whenever MANA is present, including on mixed-HCA nodes.
-HAS_MANA_RDMA_DEVICE=false
-for device in /sys/class/infiniband/*; do
-    [ -e "$device" ] || continue
-    case "$(basename "$device")" in
-        mana_*) HAS_MANA_RDMA_DEVICE=true ;;
-    esac
-done
-if [ "$HAS_MANA_RDMA_DEVICE" = "true" ]; then
-    ARGS+=(--no-collector.infiniband)
-fi
+# MANA's RDMA driver publicly supports /sys/class/infiniband, but its rate file
+# returns EINVAL with the parser used by node-exporter 1.12.1. node-exporter also
+# parses every device before applying --collector.infiniband.device-exclude, so
+# that flag cannot skip mana_* safely. Disable the whole collector when MANA is
+# present. This also suppresses metrics from other HCAs on mixed-HCA nodes until
+# upstream can filter before parsing devices.
+# https://github.com/prometheus/node_exporter/issues/3810
+# https://learn.microsoft.com/azure/virtual-network/accelerated-networking-mana-linux
+readarray -t HARDWARE_ARGS < <(getNodeExporterHardwareArgs)
+ARGS+=("${HARDWARE_ARGS[@]}")
 
 if [ -n "$TLS_CONFIG_ARG" ]; then
     ARGS+=("$TLS_CONFIG_ARG")
diff --git a/spec/parts/linux/cloud-init/artifacts/node-exporter-startup_spec.sh b/spec/parts/linux/cloud-init/artifacts/node-exporter-startup_spec.sh
new file mode 100644
index 00000000000..850ff00580e
--- /dev/null
+++ b/spec/parts/linux/cloud-init/artifacts/node-exporter-startup_spec.sh
@@ -0,0 +1,43 @@
+#!/bin/bash
+
+Describe 'node-exporter-startup.sh hardware arguments'
+    NODE_EXPORTER_STARTUP_SOURCE_ONLY=true
+    Include './parts/linux/cloud-init/artifacts/node-exporter/node-exporter-startup.sh'
+
+    setup_infiniband_class() {
+        INFINIBAND_CLASS_PATH="$(mktemp -d)"
+    }
+
+    BeforeEach 'setup_infiniband_class'
+    AfterEach 'rm -rf "$INFINIBAND_CLASS_PATH"'
+
+    It 'adds no argument without RDMA devices'
+        When call getNodeExporterHardwareArgs
+        The status should be success
+        The output should equal ''
+    End
+
+    It 'keeps the InfiniBand collector enabled for non-MANA devices'
+        mkdir "${INFINIBAND_CLASS_PATH}/mlx5_0"
+
+        When call getNodeExporterHardwareArgs
+        The status should be success
+        The output should equal ''
+    End
+
+    It 'disables the InfiniBand collector for MANA devices'
+        mkdir "${INFINIBAND_CLASS_PATH}/mana_0"
+
+        When call getNodeExporterHardwareArgs
+        The status should be success
+        The output should equal '--no-collector.infiniband'
+    End
+
+    It 'disables the InfiniBand collector on mixed MANA and non-MANA devices'
+        mkdir "${INFINIBAND_CLASS_PATH}/mana_0" "${INFINIBAND_CLASS_PATH}/mlx5_0"
+
+        When call getNodeExporterHardwareArgs
+        The status should be success
+        The output should equal '--no-collector.infiniband'
+    End
+End

From ea61c47c0112fed7d53923188bbb7a05af3e05fd Mon Sep 17 00:00:00 2001
From: chmill <chmill@microsoft.com>
Date: Fri, 4 Sep 2026 19:30:04 +0000
Subject: [PATCH 4/7] fix: detect MANA before exporter startup

---
 e2e/validators.go                             | 16 +++++++----
 .../node-exporter/node-exporter-startup.sh    | 17 ++++++-----
 .../artifacts/node-exporter-startup_spec.sh   | 28 +++++++++++++------
 3 files changed, 39 insertions(+), 22 deletions(-)

diff --git a/e2e/validators.go b/e2e/validators.go
index 5e5ac4cc62d..820bffa4064 100644
--- a/e2e/validators.go
+++ b/e2e/validators.go
@@ -2846,12 +2846,16 @@ func ValidateNodeExporter(ctx context.Context, s *Scenario) error {
 }
 
 func nodeHasInfiniBandHardware(ctx context.Context, s *Scenario) (bool, error) {
-	// MANA is an RDMA device under /sys/class/infiniband, but node-exporter 1.12.1
-	// cannot parse it and the startup script disables the collector when present.
-	// Keep this detection aligned so MANA and mixed-HCA nodes do not require
-	// metrics from a collector that must be disabled pending an upstream fix.
-	command := `for device in /sys/class/infiniband/mana_*; do
-    [ -e "$device" ] && exit 1
+	// node-exporter 1.12.1 cannot parse MANA RDMA devices, so the startup script
+	// disables the collector when MANA PCI hardware is present. Keep this
+	// detection aligned so MANA and mixed-HCA nodes do not require metrics from a
+	// collector that must be disabled pending an upstream fix.
+	command := `for device in /sys/bus/pci/devices/*; do
+    if [ -d "$device" ] &&
+       grep -qi '^0x1414$' "$device/vendor" 2>/dev/null &&
+       grep -qi '^0x00ba$' "$device/device" 2>/dev/null; then
+        exit 1
+    fi
 done
 for device in /sys/class/infiniband/*; do
     [ -e "$device" ] || continue
diff --git a/parts/linux/cloud-init/artifacts/node-exporter/node-exporter-startup.sh b/parts/linux/cloud-init/artifacts/node-exporter/node-exporter-startup.sh
index 75193cec037..d998531d64f 100755
--- a/parts/linux/cloud-init/artifacts/node-exporter/node-exporter-startup.sh
+++ b/parts/linux/cloud-init/artifacts/node-exporter/node-exporter-startup.sh
@@ -1,10 +1,12 @@
 #!/bin/bash
 
-INFINIBAND_CLASS_PATH="${INFINIBAND_CLASS_PATH:-/sys/class/infiniband}"
+PCI_DEVICES_PATH="${PCI_DEVICES_PATH:-/sys/bus/pci/devices}"
 
 getNodeExporterHardwareArgs() {
-    for device in "${INFINIBAND_CLASS_PATH}"/mana_*; do
-        if [ -e "$device" ]; then
+    for device in "${PCI_DEVICES_PATH}"/*; do
+        if [ -d "$device" ] &&
+           grep -qi '^0x1414$' "$device/vendor" 2>/dev/null &&
+           grep -qi '^0x00ba$' "$device/device" 2>/dev/null; then
             printf '%s\n' '--no-collector.infiniband'
             return
         fi
@@ -116,10 +118,11 @@ ARGS=(
 
 # MANA's RDMA driver publicly supports /sys/class/infiniband, but its rate file
 # returns EINVAL with the parser used by node-exporter 1.12.1. node-exporter also
-# parses every device before applying --collector.infiniband.device-exclude, so
-# that flag cannot skip mana_* safely. Disable the whole collector when MANA is
-# present. This also suppresses metrics from other HCAs on mixed-HCA nodes until
-# upstream can filter before parsing devices.
+# parses every device before applying either its device include or exclude
+# filter, so neither flag can avoid the failure. Detect MANA using its documented
+# PCI identity, which is available before this kubelet-ordered service starts,
+# and disable the whole collector. This also suppresses metrics from other HCAs
+# on mixed-HCA nodes until upstream can filter before parsing devices.
 # https://github.com/prometheus/node_exporter/issues/3810
 # https://learn.microsoft.com/azure/virtual-network/accelerated-networking-mana-linux
 readarray -t HARDWARE_ARGS < <(getNodeExporterHardwareArgs)
diff --git a/spec/parts/linux/cloud-init/artifacts/node-exporter-startup_spec.sh b/spec/parts/linux/cloud-init/artifacts/node-exporter-startup_spec.sh
index 850ff00580e..20f73da2031 100644
--- a/spec/parts/linux/cloud-init/artifacts/node-exporter-startup_spec.sh
+++ b/spec/parts/linux/cloud-init/artifacts/node-exporter-startup_spec.sh
@@ -4,21 +4,30 @@ Describe 'node-exporter-startup.sh hardware arguments'
     NODE_EXPORTER_STARTUP_SOURCE_ONLY=true
     Include './parts/linux/cloud-init/artifacts/node-exporter/node-exporter-startup.sh'
 
-    setup_infiniband_class() {
-        INFINIBAND_CLASS_PATH="$(mktemp -d)"
+    setup_pci_devices() {
+        PCI_DEVICES_PATH="$(mktemp -d)"
     }
 
-    BeforeEach 'setup_infiniband_class'
-    AfterEach 'rm -rf "$INFINIBAND_CLASS_PATH"'
+    add_pci_device() {
+        local name="$1"
+        local vendor="$2"
+        local device="$3"
+        mkdir "${PCI_DEVICES_PATH}/${name}"
+        printf '%s\n' "$vendor" > "${PCI_DEVICES_PATH}/${name}/vendor"
+        printf '%s\n' "$device" > "${PCI_DEVICES_PATH}/${name}/device"
+    }
+
+    BeforeEach 'setup_pci_devices'
+    AfterEach 'rm -rf "$PCI_DEVICES_PATH"'
 
-    It 'adds no argument without RDMA devices'
+    It 'adds no argument without PCI devices'
         When call getNodeExporterHardwareArgs
         The status should be success
         The output should equal ''
     End
 
-    It 'keeps the InfiniBand collector enabled for non-MANA devices'
-        mkdir "${INFINIBAND_CLASS_PATH}/mlx5_0"
+    It 'keeps the InfiniBand collector enabled for non-MANA PCI devices'
+        add_pci_device '0000:00:02.0' '0x15b3' '0x1017'
 
         When call getNodeExporterHardwareArgs
         The status should be success
@@ -26,7 +35,7 @@ Describe 'node-exporter-startup.sh hardware arguments'
     End
 
     It 'disables the InfiniBand collector for MANA devices'
-        mkdir "${INFINIBAND_CLASS_PATH}/mana_0"
+        add_pci_device '7870:00:00.0' '0x1414' '0x00ba'
 
         When call getNodeExporterHardwareArgs
         The status should be success
@@ -34,7 +43,8 @@ Describe 'node-exporter-startup.sh hardware arguments'
     End
 
     It 'disables the InfiniBand collector on mixed MANA and non-MANA devices'
-        mkdir "${INFINIBAND_CLASS_PATH}/mana_0" "${INFINIBAND_CLASS_PATH}/mlx5_0"
+        add_pci_device '0000:00:02.0' '0x15b3' '0x1017'
+        add_pci_device '7870:00:00.0' '0x1414' '0x00ba'
 
         When call getNodeExporterHardwareArgs
         The status should be success

From 4e7085b7b0f9a99cf7d4cd2703966544808af8b3 Mon Sep 17 00:00:00 2001
From: chmill <chmill@microsoft.com>
Date: Fri, 4 Sep 2026 19:39:27 +0000
Subject: [PATCH 5/7] docs: explain MANA PCI detection

---
 e2e/validators.go                                  |  2 +-
 .../node-exporter/node-exporter-startup.sh         | 14 +++++++++-----
 .../artifacts/node-exporter-startup_spec.sh        |  9 +++++++++
 3 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/e2e/validators.go b/e2e/validators.go
index 820bffa4064..02e574d903d 100644
--- a/e2e/validators.go
+++ b/e2e/validators.go
@@ -2853,7 +2853,7 @@ func nodeHasInfiniBandHardware(ctx context.Context, s *Scenario) (bool, error) {
 	command := `for device in /sys/bus/pci/devices/*; do
     if [ -d "$device" ] &&
        grep -qi '^0x1414$' "$device/vendor" 2>/dev/null &&
-       grep -qi '^0x00ba$' "$device/device" 2>/dev/null; then
+       grep -Eqi '^0x00(b9|ba|c1)$' "$device/device" 2>/dev/null; then
         exit 1
     fi
 done
diff --git a/parts/linux/cloud-init/artifacts/node-exporter/node-exporter-startup.sh b/parts/linux/cloud-init/artifacts/node-exporter/node-exporter-startup.sh
index d998531d64f..ec6c77705bb 100755
--- a/parts/linux/cloud-init/artifacts/node-exporter/node-exporter-startup.sh
+++ b/parts/linux/cloud-init/artifacts/node-exporter/node-exporter-startup.sh
@@ -6,7 +6,7 @@ getNodeExporterHardwareArgs() {
     for device in "${PCI_DEVICES_PATH}"/*; do
         if [ -d "$device" ] &&
            grep -qi '^0x1414$' "$device/vendor" 2>/dev/null &&
-           grep -qi '^0x00ba$' "$device/device" 2>/dev/null; then
+           grep -Eqi '^0x00(b9|ba|c1)$' "$device/device" 2>/dev/null; then
             printf '%s\n' '--no-collector.infiniband'
             return
         fi
@@ -119,12 +119,16 @@ ARGS=(
 # MANA's RDMA driver publicly supports /sys/class/infiniband, but its rate file
 # returns EINVAL with the parser used by node-exporter 1.12.1. node-exporter also
 # parses every device before applying either its device include or exclude
-# filter, so neither flag can avoid the failure. Detect MANA using its documented
-# PCI identity, which is available before this kubelet-ordered service starts,
-# and disable the whole collector. This also suppresses metrics from other HCAs
-# on mixed-HCA nodes until upstream can filter before parsing devices.
+# filter, so neither flag can avoid the failure. Detect MANA by its assigned PCI
+# IDs (Microsoft 1414; MANA PF 00b9, VF 00ba, PF2 00c1). PCI enumeration happens
+# during boot before userspace services start, unlike the later mana_ib driver
+# registration that creates mana_* under /sys/class/infiniband, so this check
+# cannot race RDMA class creation. Disable the whole collector when MANA is
+# present. This also suppresses metrics from other HCAs on mixed-HCA nodes until
+# upstream can filter before parsing devices.
 # https://github.com/prometheus/node_exporter/issues/3810
 # https://learn.microsoft.com/azure/virtual-network/accelerated-networking-mana-linux
+# https://github.com/torvalds/linux/blob/master/include/net/mana/gdma.h
 readarray -t HARDWARE_ARGS < <(getNodeExporterHardwareArgs)
 ARGS+=("${HARDWARE_ARGS[@]}")
 
diff --git a/spec/parts/linux/cloud-init/artifacts/node-exporter-startup_spec.sh b/spec/parts/linux/cloud-init/artifacts/node-exporter-startup_spec.sh
index 20f73da2031..6ac0dd362bc 100644
--- a/spec/parts/linux/cloud-init/artifacts/node-exporter-startup_spec.sh
+++ b/spec/parts/linux/cloud-init/artifacts/node-exporter-startup_spec.sh
@@ -42,6 +42,15 @@ Describe 'node-exporter-startup.sh hardware arguments'
         The output should equal '--no-collector.infiniband'
     End
 
+    It 'recognizes the current MANA PF and PF2 device IDs'
+        add_pci_device '7870:00:00.0' '0x1414' '0x00b9'
+        add_pci_device '7870:00:00.1' '0x1414' '0x00c1'
+
+        When call getNodeExporterHardwareArgs
+        The status should be success
+        The output should equal '--no-collector.infiniband'
+    End
+
     It 'disables the InfiniBand collector on mixed MANA and non-MANA devices'
         add_pci_device '0000:00:02.0' '0x15b3' '0x1017'
         add_pci_device '7870:00:00.0' '0x1414' '0x00ba'

From a80991200b716d86bd488d152322f2aa258abee7 Mon Sep 17 00:00:00 2001
From: chmill <chmill@microsoft.com>
Date: Tue, 8 Sep 2026 19:39:25 +0000
Subject: [PATCH 6/7] fix: handle MANA reattachment and optional exporter
 counters

---
 e2e/nodeexporter/metrics.go                   |  26 +---
 e2e/nodeexporter/metrics_test.go              |  35 +++--
 e2e/validators.go                             |  24 +++-
 .../node-exporter/node-exporter-startup.sh    |  46 +++++--
 .../artifacts/node-exporter-startup_spec.sh   | 124 +++++++++++++++++-
 vhdbuilder/packer/install-node-exporter.sh    |   7 +
 .../packer/test/linux-vhd-content-test.sh     |   5 +
 7 files changed, 225 insertions(+), 42 deletions(-)

diff --git a/e2e/nodeexporter/metrics.go b/e2e/nodeexporter/metrics.go
index 5ec6c13fc16..989f04f97aa 100644
--- a/e2e/nodeexporter/metrics.go
+++ b/e2e/nodeexporter/metrics.go
@@ -64,11 +64,14 @@ func parseMetricNames(metricsText string) map[string]struct{} {
 
 // ValidateCollectors verifies that the collectors enabled by AgentBaker are present in the scrape.
 // InfiniBand metrics are required only when the node has InfiniBand hardware.
-func ValidateCollectors(metricsText string, requireInfiniBand bool) error {
+func ValidateCollectors(metricsText string, requireInfiniBand, requireInfiniBandDisabled bool) error {
 	isVersion112OrNewer, err := nodeExporterVersionAtLeast(metricsText, 1, 12)
 	if err != nil {
 		return err
 	}
+	if requireInfiniBandDisabled && strings.Contains(metricsText, `node_scrape_collector_success{collector="infiniband"}`) {
+		return fmt.Errorf("InfiniBand collector is enabled despite the MANA workaround")
+	}
 	if isVersion112OrNewer {
 		requiredCollectors := []string{"bcachefs", "dmmultipath", "kernel_hung"}
 		var missingCollectors []string
@@ -97,25 +100,8 @@ func ValidateCollectors(metricsText string, requireInfiniBand bool) error {
 			return fmt.Errorf("InfiniBand metrics are missing")
 		}
 
-		if isVersion112OrNewer {
-			hardwareCounterMetrics := []string{
-				"node_infiniband_duplicate_requests_packets_total",
-				"node_infiniband_lifespan_seconds",
-				"node_infiniband_out_of_buffer_drops_total",
-				"node_infiniband_rx_write_requests_total",
-			}
-			hasHardwareCounterMetric := false
-			metricNames := parseMetricNames(metricsText)
-			for _, name := range hardwareCounterMetrics {
-				if _, exists := metricNames[name]; exists {
-					hasHardwareCounterMetric = true
-					break
-				}
-			}
-			if !hasHardwareCounterMetric {
-				return fmt.Errorf("InfiniBand hardware counter metrics are missing")
-			}
-		}
+		// hw_counters are optional and driver-specific, even with exporter >=1.12.
+		// Successful collection must not require counters absent from the hardware.
 	}
 
 	return nil
diff --git a/e2e/nodeexporter/metrics_test.go b/e2e/nodeexporter/metrics_test.go
index 3f4bc1a2520..7662921d36e 100644
--- a/e2e/nodeexporter/metrics_test.go
+++ b/e2e/nodeexporter/metrics_test.go
@@ -30,7 +30,7 @@ node_scrape_collector_success{collector="dmmultipath"} 1
 node_scrape_collector_success{collector="kernel_hung"} 0
 `
 
-	require.NoError(t, ValidateCollectors(metrics, false))
+	require.NoError(t, ValidateCollectors(metrics, false, false))
 }
 
 func TestValidateCollectorsRejectsMissingCollector(t *testing.T) {
@@ -39,7 +39,7 @@ node_scrape_collector_success{collector="bcachefs"} 0
 node_scrape_collector_success{collector="kernel_hung"} 0
 `
 
-	require.ErrorContains(t, ValidateCollectors(metrics, false), "dmmultipath")
+	require.ErrorContains(t, ValidateCollectors(metrics, false, false), "dmmultipath")
 }
 
 func TestValidateCollectorsRequiresInfiniBandMetricsOnInfiniBandNodes(t *testing.T) {
@@ -48,30 +48,45 @@ node_scrape_collector_success{collector="bcachefs"} 0
 node_scrape_collector_success{collector="dmmultipath"} 1
 node_scrape_collector_success{collector="kernel_hung"} 0
 node_scrape_collector_success{collector="infiniband"} 1
-node_infiniband_port_state{device="mlx5_0",port="1",state="ACTIVE"} 1
+node_infiniband_state_id{device="mlx5_0",port="1"} 4
 node_infiniband_out_of_buffer_drops_total{device="mlx5_0",port="1"} 0
 `
 
-	require.NoError(t, ValidateCollectors(metrics, true))
-	require.ErrorContains(t, ValidateCollectors(strings.Replace(metrics, "node_infiniband_", "other_metric_", -1), true), "metrics are missing")
-	require.ErrorContains(t, ValidateCollectors(strings.Replace(metrics, "\nnode_infiniband_out_of_buffer_drops_total", "\nother_metric", 1), true), "hardware counter metrics are missing")
-	require.ErrorContains(t, ValidateCollectors(strings.Replace(metrics, `collector="infiniband"} 1`, `collector="infiniband"} 0`, 1), true), "did not succeed")
+	require.NoError(t, ValidateCollectors(metrics, true, false))
+	require.ErrorContains(t, ValidateCollectors(strings.Replace(metrics, "node_infiniband_", "other_metric_", -1), true, false), "metrics are missing")
+	// A device without optional hw_counters still satisfies the common contract.
+	require.NoError(t, ValidateCollectors(strings.Replace(metrics, "node_infiniband_out_of_buffer_drops_total{device=\"mlx5_0\",port=\"1\"} 0\n", "", 1), true, false))
+	require.ErrorContains(t, ValidateCollectors(strings.Replace(metrics, `collector="infiniband"} 1`, `collector="infiniband"} 0`, 1), true, false), "did not succeed")
 }
 
 func TestValidateCollectorsSupportsMainVHDExporter(t *testing.T) {
 	metrics := `node_exporter_build_info{version="v1.9.1"} 1`
 
-	require.NoError(t, ValidateCollectors(metrics, false))
+	require.NoError(t, ValidateCollectors(metrics, false, false))
 }
 
 func TestValidateCollectorsRequiresNewCollectorsAfter112(t *testing.T) {
 	metrics := `node_exporter_build_info{version="v1.13.0"} 1`
 
-	require.ErrorContains(t, ValidateCollectors(metrics, false), "bcachefs")
+	require.ErrorContains(t, ValidateCollectors(metrics, false, false), "bcachefs")
 }
 
 func TestValidateCollectorsRejectsMissingBuildInfo(t *testing.T) {
-	require.ErrorContains(t, ValidateCollectors("", false), "build info")
+	require.ErrorContains(t, ValidateCollectors("", false, false), "build info")
+}
+
+func TestValidateCollectorsRequiresMANASuppression(t *testing.T) {
+	metrics := `node_exporter_build_info{version="v1.12.1"} 1
+node_scrape_collector_success{collector="bcachefs"} 0
+node_scrape_collector_success{collector="dmmultipath"} 1
+node_scrape_collector_success{collector="kernel_hung"} 0
+`
+	require.NoError(t, ValidateCollectors(metrics, false, true))
+	for _, value := range []string{"0", "1"} {
+		t.Run(value, func(t *testing.T) {
+			require.ErrorContains(t, ValidateCollectors(metrics+`node_scrape_collector_success{collector="infiniband"} `+value, false, true), "enabled despite the MANA workaround")
+		})
+	}
 }
 
 func validMetrics() string {
diff --git a/e2e/validators.go b/e2e/validators.go
index 02e574d903d..4dce77cc2d5 100644
--- a/e2e/validators.go
+++ b/e2e/validators.go
@@ -2847,29 +2847,47 @@ func ValidateNodeExporter(ctx context.Context, s *Scenario) error {
 
 func nodeHasInfiniBandHardware(ctx context.Context, s *Scenario) (bool, error) {
 	// node-exporter 1.12.1 cannot parse MANA RDMA devices, so the startup script
-	// disables the collector when MANA PCI hardware is present. Keep this
+	// disables the collector once MANA PCI hardware is observed this boot. Keep this
 	// detection aligned so MANA and mixed-HCA nodes do not require metrics from a
 	// collector that must be disabled pending an upstream fix.
-	command := `for device in /sys/bus/pci/devices/*; do
+	command := `if [ -f /run/node-exporter-mana-observed ]; then
+    echo "MANA observed earlier this boot; InfiniBand collector disabled"
+    exit 1
+fi
+for device in /sys/bus/pci/devices/*; do
     if [ -d "$device" ] &&
        grep -qi '^0x1414$' "$device/vendor" 2>/dev/null &&
        grep -Eqi '^0x00(b9|ba|c1)$' "$device/device" 2>/dev/null; then
+        echo "MANA PCI device: $device; InfiniBand collector disabled"
         exit 1
     fi
 done
 for device in /sys/class/infiniband/*; do
     [ -e "$device" ] || continue
+    echo "InfiniBand device: $device; requiring collector success and metrics"
     exit 0
 done
+echo "No InfiniBand devices found"
 exit 1`
 	result, err := execScriptOnVMForScenario(ctx, s, command)
 	if err != nil {
 		return false, fmt.Errorf("detect InfiniBand hardware: %w", err)
 	}
+	if result.exitCode != "0" && result.exitCode != "1" {
+		return false, fmt.Errorf("detect InfiniBand hardware: exit %s: %s", result.exitCode, result.stderr)
+	}
+	s.Logger.Logf("node-exporter hardware detection: %s", strings.TrimSpace(result.stdout))
 	return result.exitCode == "0", nil
 }
 
 func scrapeAndValidateNodeExporter(ctx context.Context, s *Scenario, metricsURL string, requireInfiniBand bool) error {
+	// The boot-local marker only exists on VHDs with lifecycle-aware suppression;
+	// main/older VHDs used by standalone PR E2Es retain their existing checks.
+	manaObserved, err := fileExist(ctx, s, "/run/node-exporter-mana-observed")
+	if err != nil {
+		return fmt.Errorf("read node-exporter MANA workaround state: %w", err)
+	}
+	s.Logger.Logf("node-exporter InfiniBand expectations: required=%t, disabled=%t", requireInfiniBand, manaObserved)
 	result, err := execScriptOnVMForScenario(ctx, s, fmt.Sprintf("curl --noproxy '*' -sS --max-time 10 %q", metricsURL))
 	if err != nil {
 		return fmt.Errorf("scrape node-exporter metrics from %s: %w", metricsURL, err)
@@ -2886,7 +2904,7 @@ func scrapeAndValidateNodeExporter(ctx context.Context, s *Scenario, metricsURL
 	}
 	return errors.Join(
 		assert.NoError(nodeexporter.ValidateMetrics(result.stdout), "node-exporter scrape did not satisfy the AKS Prometheus metrics contract\nresponse preview:\n%s", responsePreview),
-		assert.NoError(nodeexporter.ValidateCollectors(result.stdout, requireInfiniBand), "node-exporter collectors did not satisfy the AgentBaker contract\nresponse preview:\n%s", responsePreview),
+		assert.NoError(nodeexporter.ValidateCollectors(result.stdout, requireInfiniBand, manaObserved), "node-exporter collectors did not satisfy the AgentBaker contract\nresponse preview:\n%s", responsePreview),
 	)
 }
 
diff --git a/parts/linux/cloud-init/artifacts/node-exporter/node-exporter-startup.sh b/parts/linux/cloud-init/artifacts/node-exporter/node-exporter-startup.sh
index ec6c77705bb..5c90cf86fbf 100755
--- a/parts/linux/cloud-init/artifacts/node-exporter/node-exporter-startup.sh
+++ b/parts/linux/cloud-init/artifacts/node-exporter/node-exporter-startup.sh
@@ -1,22 +1,49 @@
 #!/bin/bash
 
 PCI_DEVICES_PATH="${PCI_DEVICES_PATH:-/sys/bus/pci/devices}"
+MANA_OBSERVED_FILE="${MANA_OBSERVED_FILE:-/run/node-exporter-mana-observed}"
 
 getNodeExporterHardwareArgs() {
+    if [ -f "$MANA_OBSERVED_FILE" ]; then
+        printf '%s\n' '--no-collector.infiniband'
+        return
+    fi
+    local device
     for device in "${PCI_DEVICES_PATH}"/*; do
         if [ -d "$device" ] &&
            grep -qi '^0x1414$' "$device/vendor" 2>/dev/null &&
            grep -Eqi '^0x00(b9|ba|c1)$' "$device/device" 2>/dev/null; then
+            touch "$MANA_OBSERVED_FILE" || return 1
             printf '%s\n' '--no-collector.infiniband'
             return
         fi
     done
 }
 
+nodeExporterMANAAdded() {
+    # Record the event before requesting a restart, even if the VF disappears
+    # again before ExecStart runs. Never start an inactive/preprovisioned service.
+    touch "$MANA_OBSERVED_FILE" || return 1
+    # Several VFs may arrive together. Do not restart an exporter that already
+    # applied the workaround, but do not use marker existence as proof of that.
+    local pid
+    pid=$(systemctl show --property=MainPID --value node-exporter.service) || return 1
+    if [[ "$pid" =~ ^[1-9][0-9]*$ ]] &&
+       grep -zFxq -- '--no-collector.infiniband' "/proc/${pid}/cmdline" 2>/dev/null; then
+        return 0
+    fi
+    systemctl --no-block try-restart node-exporter.service
+}
+
 if [ "${NODE_EXPORTER_STARTUP_SOURCE_ONLY:-false}" = "true" ]; then
     return 0
 fi
 
+if [ "${1:-}" = "--mana-added" ]; then
+    nodeExporterMANAAdded
+    exit $?
+fi
+
 if [ "$(grep ^ID= /etc/os-release | cut -c 4-)" = "flatcar" ]; then
     NODE_IP=$(ip -o -4 addr show dev eth0 | awk '{print $4}' | cut -d '/' -f 1)
 else
@@ -120,17 +147,20 @@ ARGS=(
 # returns EINVAL with the parser used by node-exporter 1.12.1. node-exporter also
 # parses every device before applying either its device include or exclude
 # filter, so neither flag can avoid the failure. Detect MANA by its assigned PCI
-# IDs (Microsoft 1414; MANA PF 00b9, VF 00ba, PF2 00c1). PCI enumeration happens
-# during boot before userspace services start, unlike the later mana_ib driver
-# registration that creates mana_* under /sys/class/infiniband, so this check
-# cannot race RDMA class creation. Disable the whole collector when MANA is
-# present. This also suppresses metrics from other HCAs on mixed-HCA nodes until
-# upstream can filter before parsing devices.
+# IDs (Microsoft 1414; MANA PF 00b9, VF 00ba, PF2 00c1), independently of mana_ib
+# registration. Azure servicing can remove/re-add PCI VFs after boot: remember
+# MANA in /run across service restarts and use the PCI-add udev rule installed
+# by install-node-exporter.sh to re-evaluate on late attachment. /run resets on
+# reboot and does not carry build-VM hardware observations into new nodes.
+# This suppresses all InfiniBand metrics, including other HCAs on mixed nodes,
+# until upstream supports filtering before parsing devices.
 # https://github.com/prometheus/node_exporter/issues/3810
 # https://learn.microsoft.com/azure/virtual-network/accelerated-networking-mana-linux
 # https://github.com/torvalds/linux/blob/master/include/net/mana/gdma.h
-readarray -t HARDWARE_ARGS < <(getNodeExporterHardwareArgs)
-ARGS+=("${HARDWARE_ARGS[@]}")
+HARDWARE_ARG=$(getNodeExporterHardwareArgs) || exit 1
+if [ -n "$HARDWARE_ARG" ]; then
+    ARGS+=("$HARDWARE_ARG")
+fi
 
 if [ -n "$TLS_CONFIG_ARG" ]; then
     ARGS+=("$TLS_CONFIG_ARG")
diff --git a/spec/parts/linux/cloud-init/artifacts/node-exporter-startup_spec.sh b/spec/parts/linux/cloud-init/artifacts/node-exporter-startup_spec.sh
index 6ac0dd362bc..54fdf3c777d 100644
--- a/spec/parts/linux/cloud-init/artifacts/node-exporter-startup_spec.sh
+++ b/spec/parts/linux/cloud-init/artifacts/node-exporter-startup_spec.sh
@@ -6,6 +6,7 @@ Describe 'node-exporter-startup.sh hardware arguments'
 
     setup_pci_devices() {
         PCI_DEVICES_PATH="$(mktemp -d)"
+        MANA_OBSERVED_FILE="${PCI_DEVICES_PATH}/mana-observed"
     }
 
     add_pci_device() {
@@ -42,8 +43,15 @@ Describe 'node-exporter-startup.sh hardware arguments'
         The output should equal '--no-collector.infiniband'
     End
 
-    It 'recognizes the current MANA PF and PF2 device IDs'
+    It 'recognizes the MANA PF device ID'
         add_pci_device '7870:00:00.0' '0x1414' '0x00b9'
+
+        When call getNodeExporterHardwareArgs
+        The status should be success
+        The output should equal '--no-collector.infiniband'
+    End
+
+    It 'recognizes the MANA PF2 device ID'
         add_pci_device '7870:00:00.1' '0x1414' '0x00c1'
 
         When call getNodeExporterHardwareArgs
@@ -51,6 +59,14 @@ Describe 'node-exporter-startup.sh hardware arguments'
         The output should equal '--no-collector.infiniband'
     End
 
+    It 'does not match another vendor with the same device ID'
+        add_pci_device '7870:00:00.0' '0x15b3' '0x00ba'
+
+        When call getNodeExporterHardwareArgs
+        The status should be success
+        The output should equal ''
+    End
+
     It 'disables the InfiniBand collector on mixed MANA and non-MANA devices'
         add_pci_device '0000:00:02.0' '0x15b3' '0x1017'
         add_pci_device '7870:00:00.0' '0x1414' '0x00ba'
@@ -59,4 +75,110 @@ Describe 'node-exporter-startup.sh hardware arguments'
         The status should be success
         The output should equal '--no-collector.infiniband'
     End
+
+    run_startup() {
+        NODE_EXPORTER_STARTUP_SOURCE_ONLY=false
+        NODE_EXPORTER_TLS_ENABLED=false
+        hostname() { printf '%s\n' '192.0.2.1'; }
+        exec() { printf '%s\n' "$@"; }
+        . ./parts/linux/cloud-init/artifacts/node-exporter/node-exporter-startup.sh
+    }
+
+    It 'passes the hardware flag to the final exporter invocation'
+        add_pci_device '7870:00:00.0' '0x1414' '0x00ba'
+
+        When run run_startup
+        The status should be success
+        The line 1 of output should equal '/opt/bin/node-exporter'
+        The output should include '--web.listen-address=192.0.2.1:19100'
+        The output should include '--no-collector.infiniband'
+        The path "$MANA_OBSERVED_FILE" should be file
+    End
+
+    It 'leaves InfiniBand enabled in the final invocation without MANA'
+        When run run_startup
+        The status should be success
+        The output should include '/opt/bin/node-exporter'
+        The output should not include '--no-collector.infiniband'
+    End
+
+    restart_during_vf_absence() {
+        getNodeExporterHardwareArgs >/dev/null
+        rm -rf "${PCI_DEVICES_PATH}/7870:00:00.0"
+        run_startup
+    }
+
+    It 'keeps suppression after an observed VF disappears and exporter restarts'
+        add_pci_device '7870:00:00.0' '0x1414' '0x00ba'
+
+        When run restart_during_vf_absence
+        The status should be success
+        The output should include '--no-collector.infiniband'
+    End
+
+    systemctl() {
+        # The event must be recorded before systemd can run the next ExecStart.
+        [ -f "$MANA_OBSERVED_FILE" ] || return 1
+        if [ "$1" = 'show' ]; then
+            printf '%s\n' '0'
+        else
+            printf '%s\n' "$*"
+        fi
+    }
+
+    run_attach_handler() {
+        NODE_EXPORTER_STARTUP_SOURCE_ONLY=false
+        . ./parts/linux/cloud-init/artifacts/node-exporter/node-exporter-startup.sh --mana-added
+    }
+
+    It 'records late attachment and queues only a nonblocking conditional restart'
+        When run run_attach_handler
+        The status should be success
+        The output should equal '--no-block try-restart node-exporter.service'
+        The path "$MANA_OBSERVED_FILE" should be file
+    End
+
+    attach_after_startup() {
+        run_startup
+        nodeExporterMANAAdded
+        # The event marker is sufficient even if the VF vanishes before restart.
+        run_startup
+    }
+
+    It 'disables collection on restart after attachment was missed at startup'
+        When run attach_after_startup
+        The status should be success
+        The output should include '--no-block try-restart node-exporter.service'
+        The output should include '--no-collector.infiniband'
+    End
+
+    It 'does not request a restart if recording the attachment fails'
+        MANA_OBSERVED_FILE="${PCI_DEVICES_PATH}/missing/marker"
+
+        When run run_attach_handler
+        The status should be failure
+        The stderr should include 'No such file or directory'
+        The output should equal ''
+    End
+
+    It 'does not restart again when the running exporter already disabled InfiniBand'
+        systemctl() {
+            if [ "$1" = 'show' ]; then printf '%s\n' '123'; else return 1; fi
+        }
+        grep() {
+            [ "$*" = '-zFxq -- --no-collector.infiniband /proc/123/cmdline' ]
+        }
+
+        When run run_attach_handler
+        The status should be success
+        The output should equal ''
+    End
+
+    It 'requests a restart even with an existing marker if startup has not applied it'
+        touch "$MANA_OBSERVED_FILE"
+
+        When run run_attach_handler
+        The status should be success
+        The output should equal '--no-block try-restart node-exporter.service'
+    End
 End
diff --git a/vhdbuilder/packer/install-node-exporter.sh b/vhdbuilder/packer/install-node-exporter.sh
index 135dd4e382d..978406acc9f 100644
--- a/vhdbuilder/packer/install-node-exporter.sh
+++ b/vhdbuilder/packer/install-node-exporter.sh
@@ -37,6 +37,13 @@ installNodeExporter() {
     systemctl daemon-reload
     systemctl disable node-exporter.service node-exporter-restart.path || exit 1
 
+    # VFs can arrive after exporter startup or return after Azure host servicing.
+    # The handler records MANA for this boot and restarts only an active exporter.
+    mkdir -p /etc/udev/rules.d
+    printf '%s\n' 'ACTION=="add", SUBSYSTEM=="pci", ATTR{vendor}=="0x1414", ATTR{device}=="0x00b9|0x00ba|0x00c1", RUN+="/opt/bin/node-exporter-startup.sh --mana-added"' \
+        > /etc/udev/rules.d/99-node-exporter-mana.rules
+    udevadm control --reload-rules
+
     # Create skip sentinel file to indicate node-exporter was installed from VHD
     mkdir -p /etc/node-exporter.d
     touch /etc/node-exporter.d/skip_vhd_node_exporter
diff --git a/vhdbuilder/packer/test/linux-vhd-content-test.sh b/vhdbuilder/packer/test/linux-vhd-content-test.sh
index 1c0d214080a..8e5814ed440 100644
--- a/vhdbuilder/packer/test/linux-vhd-content-test.sh
+++ b/vhdbuilder/packer/test/linux-vhd-content-test.sh
@@ -2039,6 +2039,11 @@ testNodeExporter () {
   fi
   echo "$test: node-exporter startup script exists"
 
+  if [ ! -s /etc/udev/rules.d/99-node-exporter-mana.rules ]; then
+    err "$test" "node-exporter MANA PCI-add rule is missing"
+    return 1
+  fi
+
   # Check that the service file exists
   if [ ! -f "/etc/systemd/system/node-exporter.service" ]; then
     err "$test" "node-exporter service file does not exist at /etc/systemd/system/node-exporter.service"

From a6aa6ddeddd40c24f5e1be5e698bcb5e1ea37d56 Mon Sep 17 00:00:00 2001
From: chmill <chmill@microsoft.com>
Date: Tue, 8 Sep 2026 20:34:41 +0000
Subject: [PATCH 7/7] fix: enforce MANA exporter arguments and portable PID
 check

---
 .../node-exporter/node-exporter-startup.sh    | 30 +++++++++++-----
 .../artifacts/node-exporter-startup_spec.sh   | 35 +++++++++++++++++++
 2 files changed, 57 insertions(+), 8 deletions(-)

diff --git a/parts/linux/cloud-init/artifacts/node-exporter/node-exporter-startup.sh b/parts/linux/cloud-init/artifacts/node-exporter/node-exporter-startup.sh
index 5c90cf86fbf..40ad8cf5f0f 100755
--- a/parts/linux/cloud-init/artifacts/node-exporter/node-exporter-startup.sh
+++ b/parts/linux/cloud-init/artifacts/node-exporter/node-exporter-startup.sh
@@ -28,10 +28,14 @@ nodeExporterMANAAdded() {
     # applied the workaround, but do not use marker existence as proof of that.
     local pid
     pid=$(systemctl show --property=MainPID --value node-exporter.service) || return 1
-    if [[ "$pid" =~ ^[1-9][0-9]*$ ]] &&
-       grep -zFxq -- '--no-collector.infiniband' "/proc/${pid}/cmdline" 2>/dev/null; then
-        return 0
-    fi
+    case "$pid" in
+        ''|0*|*[!0-9]*) ;;
+        *)
+            if grep -zFxq -- '--no-collector.infiniband' "/proc/${pid}/cmdline" 2>/dev/null; then
+                return 0
+            fi
+            ;;
+    esac
     systemctl --no-block try-restart node-exporter.service
 }
 
@@ -158,9 +162,6 @@ ARGS=(
 # https://learn.microsoft.com/azure/virtual-network/accelerated-networking-mana-linux
 # https://github.com/torvalds/linux/blob/master/include/net/mana/gdma.h
 HARDWARE_ARG=$(getNodeExporterHardwareArgs) || exit 1
-if [ -n "$HARDWARE_ARG" ]; then
-    ARGS+=("$HARDWARE_ARG")
-fi
 
 if [ -n "$TLS_CONFIG_ARG" ]; then
     ARGS+=("$TLS_CONFIG_ARG")
@@ -170,7 +171,20 @@ fi
 # Example: NODE_EXPORTER_EXTRA_ARGS="--collector.systemd --no-collector.bonding"
 if [ -n "${NODE_EXPORTER_EXTRA_ARGS:-}" ]; then
     read -ra EXTRA <<< "$NODE_EXPORTER_EXTRA_ARGS"
-    ARGS+=("${EXTRA[@]}")
+    for arg in "${EXTRA[@]}"; do
+        if [ -n "$HARDWARE_ARG" ]; then
+            case "$arg" in
+                --collector.infiniband|--collector.infiniband=*|--no-collector.infiniband|--no-collector.infiniband=*) continue ;;
+            esac
+        fi
+        ARGS+=("$arg")
+    done
+fi
+
+# Kingpin rejects repeated flags. Remove conflicting optional toggles above and
+# append exactly one mandatory override so extra arguments cannot undo it.
+if [ -n "$HARDWARE_ARG" ]; then
+    ARGS+=("$HARDWARE_ARG")
 fi
 
 exec /opt/bin/node-exporter "${ARGS[@]}"
diff --git a/spec/parts/linux/cloud-init/artifacts/node-exporter-startup_spec.sh b/spec/parts/linux/cloud-init/artifacts/node-exporter-startup_spec.sh
index 54fdf3c777d..0f1e460817a 100644
--- a/spec/parts/linux/cloud-init/artifacts/node-exporter-startup_spec.sh
+++ b/spec/parts/linux/cloud-init/artifacts/node-exporter-startup_spec.sh
@@ -7,6 +7,7 @@ Describe 'node-exporter-startup.sh hardware arguments'
     setup_pci_devices() {
         PCI_DEVICES_PATH="$(mktemp -d)"
         MANA_OBSERVED_FILE="${PCI_DEVICES_PATH}/mana-observed"
+        NODE_EXPORTER_EXTRA_ARGS=''
     }
 
     add_pci_device() {
@@ -102,6 +103,28 @@ Describe 'node-exporter-startup.sh hardware arguments'
         The output should not include '--no-collector.infiniband'
     End
 
+    It 'replaces conflicting InfiniBand toggles with exactly one MANA override'
+        add_pci_device '7870:00:00.0' '0x1414' '0x00ba'
+        NODE_EXPORTER_EXTRA_ARGS='--collector.infiniband --collector.infiniband=true --no-collector.infiniband --no-collector.infiniband=false --collector.systemd'
+
+        When run run_startup
+        The status should be success
+        The output should not include '--collector.infiniband'
+        The output should not include '--no-collector.infiniband='
+        The output should include '--collector.systemd'
+        The output should end with '--no-collector.infiniband'
+        The lines of output should equal 12
+    End
+
+    It 'preserves explicit InfiniBand arguments on non-MANA nodes'
+        NODE_EXPORTER_EXTRA_ARGS='--collector.infiniband'
+
+        When run run_startup
+        The status should be success
+        The output should end with '--collector.infiniband'
+        The output should not include '--no-collector.infiniband'
+    End
+
     restart_during_vf_absence() {
         getNodeExporterHardwareArgs >/dev/null
         rm -rf "${PCI_DEVICES_PATH}/7870:00:00.0"
@@ -181,4 +204,16 @@ Describe 'node-exporter-startup.sh hardware arguments'
         The status should be success
         The output should equal '--no-block try-restart node-exporter.service'
     End
+
+    It 'does not inspect procfs for an invalid MainPID'
+        systemctl() {
+            if [ "$1" = 'show' ]; then printf '%s\n' '../123'; else printf '%s\n' "$*"; fi
+        }
+        grep() { printf '%s\n' 'unexpected procfs read' >&2; return 1; }
+
+        When run run_attach_handler
+        The status should be success
+        The output should equal '--no-block try-restart node-exporter.service'
+        The stderr should equal ''
+    End
 End