From cca4f1c9ff79e21e1237dd2b19866d9034cbf3c1 Mon Sep 17 00:00:00 2001 From: Mitch Zhu Date: Tue, 1 Sep 2026 18:26:37 +0000 Subject: [PATCH 1/3] feat(vhd): select Azure Linux ARM64 kernel by hardware Install standard and HWE kernels in the Azure Linux 3 ARM64 image. Select HWE with Grace boot arguments only on NVIDIA Grace hardware while ordinary ARM64 keeps the standard kernel. Signed-off-by: Mitch Zhu --- .../cloud-init/artifacts/10_azure_nvidia | 57 ++++++++++++--- .../cloud-init/artifacts/51-azure-nvidia.cfg | 4 +- .../cloud-init/artifacts/azure_nvidia_spec.sh | 73 +++++++++++++++++++ vhdbuilder/packer/packer_source.sh | 10 +++ vhdbuilder/packer/pre-install-dependencies.sh | 61 ++++++++++++++++ .../vhd-image-builder-mariner-arm64.json | 10 +++ 6 files changed, 201 insertions(+), 14 deletions(-) create mode 100644 spec/parts/linux/cloud-init/artifacts/azure_nvidia_spec.sh diff --git a/parts/linux/cloud-init/artifacts/10_azure_nvidia b/parts/linux/cloud-init/artifacts/10_azure_nvidia index 47d5a2fab15..301c21e7d69 100755 --- a/parts/linux/cloud-init/artifacts/10_azure_nvidia +++ b/parts/linux/cloud-init/artifacts/10_azure_nvidia @@ -2,30 +2,63 @@ set -e -. /etc/grub.d/10_linux >/dev/null 2>&1 +BOOT_DIR="${BOOT_DIR:-/boot}" +GRUB_LINUX_SCRIPT="${GRUB_LINUX_SCRIPT:-/etc/grub.d/10_linux}" + +boot_device_id= +reverse_sorted_list= +if [ -r "$GRUB_LINUX_SCRIPT" ]; then + # shellcheck source=/etc/grub.d/10_linux + # shellcheck disable=SC1091 + . "$GRUB_LINUX_SCRIPT" >/dev/null 2>&1 +fi NVIDIA= OTHER= -for linux in ${reverse_sorted_list}; do - case $linux in - *-azure-nvidia) : "${NVIDIA:="${linux}"}" ;; - *) : "${OTHER:="${linux}"}" ;; - esac -done +# Azure Linux kernel filenames do not identify their track, so use RPM ownership. +if command -v rpm >/dev/null 2>&1; then + for linux in "${BOOT_DIR}"/vmlinuz-*; do + [ -f "$linux" ] || continue + package=$(rpm -qf --queryformat '%{NAME}' "$linux" 2>/dev/null || true) + + case $package in + kernel-hwe) NVIDIA=$(printf '%s\n%s\n' "$NVIDIA" "$linux" | sed '/^$/d' | sort -V | tail -n1) ;; + kernel) OTHER=$(printf '%s\n%s\n' "$OTHER" "$linux" | sed '/^$/d' | sort -V | tail -n1) ;; + esac + done +fi + +# Ubuntu identifies its NVIDIA kernel by filename and exposes a sorted list from 10_linux. +if [ -z "$NVIDIA" ] || [ -z "$OTHER" ]; then + NVIDIA= + OTHER= + # shellcheck disable=SC2086 # reverse_sorted_list is a space-separated path list from 10_linux. + for linux in ${reverse_sorted_list}; do + case $linux in + *-azure-nvidia) : "${NVIDIA:="${linux}"}" ;; + *) : "${OTHER:="${linux}"}" ;; + esac + done +fi -if [ -z "${NVIDIA}" ] || [ -z "${OTHER}" ]; then +if [ -z "$NVIDIA" ] || [ -z "$OTHER" ]; then echo "Only one image type (NVIDIA or non-NVIDIA) found" >&2 exit 0 fi -echo "Default NVIDIA image: ${NVIDIA}" >&2 NVIDIA=${NVIDIA##*/} -NVIDIA=$(echo "${NVIDIA}" | sed -e "s,^[^0-9]*-,,g") +NVIDIA=${NVIDIA#vmlinuz-} -echo "Default non-NVIDIA image: ${OTHER}" >&2 OTHER=${OTHER##*/} -OTHER=$(echo "${OTHER}" | sed -e "s,^[^0-9]*-,,g") +OTHER=${OTHER#vmlinuz-} + +if [ -z "$boot_device_id" ]; then + echo "Could not determine boot_device_id from $GRUB_LINUX_SCRIPT" >&2 + exit 0 +fi + +echo "Default NVIDIA image: $NVIDIA, non-NVIDIA image: $OTHER, boot_device_id: $boot_device_id" >&2 cat << EOF insmod smbios diff --git a/parts/linux/cloud-init/artifacts/51-azure-nvidia.cfg b/parts/linux/cloud-init/artifacts/51-azure-nvidia.cfg index 97b32adcd77..bfae6ba7671 100644 --- a/parts/linux/cloud-init/artifacts/51-azure-nvidia.cfg +++ b/parts/linux/cloud-init/artifacts/51-azure-nvidia.cfg @@ -1,5 +1,5 @@ -# 10_azure_nvidia changes the default menu entry based on a flat menu. +# 10_azure_nvidia selects the hardware-appropriate entry from a flat menu. GRUB_DISABLE_SUBMENU=true -# 10_azure_nvidia appends additional arguments for azure-nvidia kernels. +# Append NVIDIA Grace arguments only when the selector sets nvidia_args. GRUB_CMDLINE_LINUX="$GRUB_CMDLINE_LINUX \$nvidia_args" diff --git a/spec/parts/linux/cloud-init/artifacts/azure_nvidia_spec.sh b/spec/parts/linux/cloud-init/artifacts/azure_nvidia_spec.sh new file mode 100644 index 00000000000..e468b4c39e3 --- /dev/null +++ b/spec/parts/linux/cloud-init/artifacts/azure_nvidia_spec.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +Describe '10_azure_nvidia' + SCRIPT='./parts/linux/cloud-init/artifacts/10_azure_nvidia' + + Mock rpm + for arg do + kernel_path=$arg + done + case "${kernel_path##*/}" in + vmlinuz-1.0.*) printf 'kernel' ;; + vmlinuz-2.0.*) printf 'kernel-hwe' ;; + *) exit 1 ;; + esac + End + + setup() { + TEST_DIR=$(mktemp -d) + BOOT_DIR="${TEST_DIR}/boot" + GRUB_LINUX_SCRIPT="${TEST_DIR}/10_linux" + mkdir -p "$BOOT_DIR" + for kernel in 1.0.9-test 1.0.10-test 2.0.9-test 2.0.10-test 9.0.0-unowned; do + touch "${BOOT_DIR}/vmlinuz-${kernel}" + done + printf '%s\n' 'boot_device_id=test-device' > "$GRUB_LINUX_SCRIPT" + export BOOT_DIR GRUB_LINUX_SCRIPT + } + + cleanup() { + rm -rf "$TEST_DIR" + } + + BeforeEach 'setup' + AfterEach 'cleanup' + + It 'selects the newest RPM-owned kernel from each Azure Linux track' + When run script "$SCRIPT" + The status should be success + The output should include 'set default="gnulinux-2.0.10-test-advanced-test-device"' + The output should include 'set default="gnulinux-1.0.10-test-advanced-test-device"' + The output should not include '9.0.0-unowned' + The stderr should include 'Default NVIDIA image: 2.0.10-test, non-NVIDIA image: 1.0.10-test' + End + + It 'preserves Ubuntu kernel selection by filename' + rm -f "${BOOT_DIR}"/vmlinuz-* + cat > "$GRUB_LINUX_SCRIPT" <<'EOF' +boot_device_id=test-device +reverse_sorted_list='/boot/vmlinuz-3.0.10-azure-nvidia /boot/vmlinuz-3.0.10-azure' +EOF + When run script "$SCRIPT" + The status should be success + The output should include 'set default="gnulinux-3.0.10-azure-nvidia-advanced-test-device"' + The output should include 'set default="gnulinux-3.0.10-azure-advanced-test-device"' + The stderr should include 'Default NVIDIA image: 3.0.10-azure-nvidia, non-NVIDIA image: 3.0.10-azure' + End + + It 'emits no override when either kernel track is missing' + rm -f "${BOOT_DIR}"/vmlinuz-2.0.* + When run script "$SCRIPT" + The status should be success + The output should eq '' + The stderr should include 'Only one image type (NVIDIA or non-NVIDIA) found' + End + + It 'emits no override when 10_linux cannot derive a boot device ID' + printf '%s\n' 'boot_device_id=' > "$GRUB_LINUX_SCRIPT" + When run script "$SCRIPT" + The status should be success + The output should eq '' + The stderr should include "Could not determine boot_device_id from $GRUB_LINUX_SCRIPT" + End +End \ No newline at end of file diff --git a/vhdbuilder/packer/packer_source.sh b/vhdbuilder/packer/packer_source.sh index e6b73484d4b..a5e98d364b1 100644 --- a/vhdbuilder/packer/packer_source.sh +++ b/vhdbuilder/packer/packer_source.sh @@ -611,6 +611,16 @@ copyPackerFiles() { # Mariner/AzureLinux uses system-auth and system-password instead of common-auth and common-password. cpAndMode $PAM_D_SYSTEM_AUTH_SRC $PAM_D_SYSTEM_AUTH_DEST 644 cpAndMode $PAM_D_SYSTEM_PASSWORD_SRC $PAM_D_SYSTEM_PASSWORD_DEST 644 + + if [ "$OS" = "$AZURELINUX_OS_NAME" ] && [ "$OS_VERSION" = "3.0" ] && [ "$CPU_ARCH" = "arm64" ] && [ -z "$OS_VARIANT" ] && [ "${ENABLE_FIPS,,}" != "true" ]; then + GRUB_AZ_NV_SCRIPT_SRC=/home/packer/10_azure_nvidia + GRUB_AZ_NV_SCRIPT_DEST=/etc/grub.d/10_azure_nvidia + cpAndMode $GRUB_AZ_NV_SCRIPT_SRC $GRUB_AZ_NV_SCRIPT_DEST 755 + + GRUB_AZ_NV_ENV_SRC=/home/packer/51-azure-nvidia.cfg + GRUB_AZ_NV_ENV_DEST=/etc/default/grub.d/51-azure-nvidia.cfg + cpAndMode $GRUB_AZ_NV_ENV_SRC $GRUB_AZ_NV_ENV_DEST 644 + fi elif isACL "$OS" "$OS_VARIANT"; then # ACL cannot share the isMarinerOrAzureLinux block because: # - containerd.service: ACL provides containerd via sysext. diff --git a/vhdbuilder/packer/pre-install-dependencies.sh b/vhdbuilder/packer/pre-install-dependencies.sh index 7557baa1442..9c35a76a9bb 100644 --- a/vhdbuilder/packer/pre-install-dependencies.sh +++ b/vhdbuilder/packer/pre-install-dependencies.sh @@ -172,6 +172,67 @@ if isMarinerOrAzureLinux "$OS" && [ "$OS_VERSION" = "3.0" ]; then fi capture_benchmark "${SCRIPT_NAME}_disable_kernel_lockdown_cmdline" +# Azure Linux currently marks kernel and kernel-hwe as conflicting even though +# their boot files do not overlap. Use dnf5 to co-install them until that package +# conflict is removed upstream. +if [ "$OS" = "$AZURELINUX_OS_NAME" ] && [ "$OS_VERSION" = "3.0" ] && [ "$CPU_ARCH" = "arm64" ] && [ -z "$OS_VARIANT" ] && [ "${ENABLE_FIPS,,}" != "true" ]; then + if ! rpm -q kernel-hwe &>/dev/null; then + dnf5_was_installed=false + if rpm -q dnf5 &>/dev/null; then + dnf5_was_installed=true + else + dnf_install 30 1 600 dnf5 || exit "$ERR_APT_INSTALL_TIMEOUT" + fi + + dnf5 install -y kernel-hwe || exit "$ERR_APT_INSTALL_TIMEOUT" + + if ! $dnf5_was_installed; then + tdnf remove -y dnf5 || true + fi + fi + + for kernel_package in kernel kernel-hwe; do + if ! rpm -q "$kernel_package" &>/dev/null || ! rpm -ql "$kernel_package" | grep -q '^/boot/vmlinuz-'; then + echo "ARM64 Azure Linux: $kernel_package does not provide a bootable kernel" >&2 + exit "$ERR_APT_INSTALL_TIMEOUT" + fi + done + + # The signed ARM64 EFI binary does not embed smbios, and grub2-efi installs + # its dynamic modules outside the boot prefix. Stage the required closure + # until Azure Linux provides smbios at boot directly. + dnf_install 30 1 600 grub2-efi || exit "$ERR_APT_INSTALL_TIMEOUT" + grub_version=$(rpm -q --queryformat '%{VERSION}-%{RELEASE}' grub2) + grub_efi_binary_version=$(rpm -q --queryformat '%{VERSION}-%{RELEASE}' grub2-efi-binary) + grub_efi_modules_version=$(rpm -q --queryformat '%{VERSION}-%{RELEASE}' grub2-efi) + if [ "$grub_version" != "$grub_efi_binary_version" ] || [ "$grub_version" != "$grub_efi_modules_version" ]; then + echo "ARM64 Azure Linux: GRUB package versions do not match: grub2=$grub_version, binary=$grub_efi_binary_version, modules=$grub_efi_modules_version" >&2 + exit "$ERR_APT_INSTALL_TIMEOUT" + fi + + grub_module_source=/usr/lib/grub/arm64-efi + grub_module_destination=/boot/grub2/arm64-efi + for grub_module_file in extcmd.mod smbios.mod moddep.lst; do + if [ ! -s "$grub_module_source/$grub_module_file" ]; then + echo "ARM64 Azure Linux: required GRUB file $grub_module_source/$grub_module_file is missing" >&2 + exit "$ERR_APT_INSTALL_TIMEOUT" + fi + done + if ! grep -q '^smbios: extcmd$' "$grub_module_source/moddep.lst"; then + echo "ARM64 Azure Linux: unexpected smbios module dependencies" >&2 + exit "$ERR_APT_INSTALL_TIMEOUT" + fi + install -d -m 0755 "$grub_module_destination" + install -m 0644 \ + "$grub_module_source/extcmd.mod" \ + "$grub_module_source/smbios.mod" \ + "$grub_module_source/moddep.lst" \ + "$grub_module_destination/" + + grub2-mkconfig -o /boot/grub2/grub.cfg +fi +capture_benchmark "${SCRIPT_NAME}_install_kernel_hwe_arm64" + # shellcheck disable=SC3010 if [[ ${UBUNTU_RELEASE//./} -ge 2204 && "${ENABLE_FIPS,,}" != "true" ]]; then diff --git a/vhdbuilder/packer/vhd-image-builder-mariner-arm64.json b/vhdbuilder/packer/vhd-image-builder-mariner-arm64.json index eb2a1aab9e6..d35c9323b2a 100644 --- a/vhdbuilder/packer/vhd-image-builder-mariner-arm64.json +++ b/vhdbuilder/packer/vhd-image-builder-mariner-arm64.json @@ -401,6 +401,16 @@ "source": "vhdbuilder/packer/post-install-dependencies.sh", "destination": "/home/packer/post-install-dependencies.sh" }, + { + "type": "file", + "source": "parts/linux/cloud-init/artifacts/10_azure_nvidia", + "destination": "/home/packer/10_azure_nvidia" + }, + { + "type": "file", + "source": "parts/linux/cloud-init/artifacts/51-azure-nvidia.cfg", + "destination": "/home/packer/51-azure-nvidia.cfg" + }, { "type": "file", "source": "parts/common/components.json", From 865cb7aaf4932c07ab5c69488b28865b230ac2c1 Mon Sep 17 00:00:00 2001 From: Mitch Zhu Date: Tue, 1 Sep 2026 18:26:59 +0000 Subject: [PATCH 2/3] feat(cse): support managed GPU drivers on Azure Linux ARM64 Enable standard non-FIPS Azure Linux 3 ARM64 while preserving the existing install-or-validate dispatch. Resolve cuda-open packages against the running standard or HWE kernel from current repository state. Signed-off-by: Mitch Zhu --- .../linux/cloud-init/artifacts/cse_config.sh | 8 ++- .../artifacts/mariner/cse_install_mariner.sh | 9 ++-- pkg/agent/baker_test.go | 11 +++++ .../cloud-init/artifacts/cse_config_spec.sh | 49 +++++++++++++++++++ .../artifacts/cse_install_mariner_spec.sh | 22 ++++++++- 5 files changed, 88 insertions(+), 11 deletions(-) diff --git a/parts/linux/cloud-init/artifacts/cse_config.sh b/parts/linux/cloud-init/artifacts/cse_config.sh index 2e9344cf0ab..2ab8aded512 100755 --- a/parts/linux/cloud-init/artifacts/cse_config.sh +++ b/parts/linux/cloud-init/artifacts/cse_config.sh @@ -1553,10 +1553,6 @@ configGPUDrivers() { } validateGPUDrivers() { - if [ "$(isARM64)" -eq 1 ]; then - return - fi - retrycmd_if_failure 24 5 25 nvidia-modprobe -u -c0 && echo "gpu driver loaded" || configGPUDrivers || exit $ERR_GPU_DRIVERS_START_FAIL if which nvidia-smi; then @@ -1635,7 +1631,9 @@ cleanUpGridNodeCudaPrebake() { ensureGPUDrivers() { if [ "$(isARM64)" -eq 1 ]; then - return + if [ "$OS" != "$AZURELINUX_OS_NAME" ] || [ "$OS_VERSION" != "3.0" ] || [ -n "$OS_VARIANT" ] || [ "${ENABLE_FIPS,,}" = "true" ]; then + return + fi fi # Tear down a mismatched cuda-lts VHD prebake before a GRID node installs its own driver, or the diff --git a/parts/linux/cloud-init/artifacts/mariner/cse_install_mariner.sh b/parts/linux/cloud-init/artifacts/mariner/cse_install_mariner.sh index 5270cc6d942..5483ca84b55 100755 --- a/parts/linux/cloud-init/artifacts/mariner/cse_install_mariner.sh +++ b/parts/linux/cloud-init/artifacts/mariner/cse_install_mariner.sh @@ -116,6 +116,7 @@ getAzureLinuxNvidiaDriverVersionFromPackage() { local version_with_epoch version_with_epoch=${package#"${package_prefix}"} + version_with_epoch=${version_with_epoch#hwe-} version_with_epoch=${version_with_epoch%%-*} echo "${version_with_epoch#*:}" } @@ -127,7 +128,7 @@ getAzureLinuxNvidiaDriverReleaseNotes() { local cuda_open_package local cuda_package local grid_package - cuda_open_package=$(getLatestAzureLinuxNvidiaDriverPackageForKernel "cuda-open*" "^cuda-open-[0-9]" "${kernel_version}") + cuda_open_package=$(getLatestAzureLinuxNvidiaDriverPackageForKernel "cuda-open*" "^cuda-open(-hwe)?-[0-9]" "${kernel_version}") cuda_package=$(getLatestAzureLinuxNvidiaDriverPackageForKernel "cuda" "^cuda-[0-9]" "${kernel_version}") grid_package=$(getLatestAzureLinuxNvidiaDriverPackageForKernel "nvidia-vgpu-guest-driver*" "^nvidia-vgpu-guest-driver-[0-9]" "${kernel_version}") @@ -155,7 +156,7 @@ downloadGPUDrivers() { # cuda-%{nvidia gpu driver version}_%{kernel source version}.%{kernel release version}.{mariner rpm postfix} # # 2. NVIDIA OpenRM driver: - # cuda-open-%{nvidia gpu driver version}_%{kernel source version}.%{kernel release version}.{mariner rpm postfix} + # cuda-open[-hwe]-%{nvidia gpu driver version}_%{kernel source version}.%{kernel release version}.{mariner rpm postfix} # # 3. NVIDIA GRID (vGPU guest) driver for converged GPU sizes: # nvidia-vgpu-guest-driver-%{version}_%{kernel version}.{mariner rpm postfix} @@ -193,10 +194,10 @@ downloadGPUDrivers() { exit $ERR_MISSING_CUDA_PACKAGE elif [ "$driver_ret" -eq 0 ]; then echo "VM SKU ${VM_SKU} uses NVIDIA OpenRM driver (cuda-open)" - CUDA_PACKAGE=$(dnf repoquery -y --available "cuda-open*" | grep -E "^cuda-open-[0-9]+.*_${KERNEL_VERSION}" | sort -V | tail -n 1) + CUDA_PACKAGE=$(getLatestAzureLinuxNvidiaDriverPackageForKernel "cuda-open*" "^cuda-open(-hwe)?-[0-9]" "$KERNEL_VERSION") else echo "VM SKU ${VM_SKU} uses NVIDIA proprietary driver (cuda)" - CUDA_PACKAGE=$(dnf repoquery -y --available "cuda-[0-9]*" | grep -E "^cuda-[0-9]+.*_${KERNEL_VERSION}" | sort -V | tail -n 1) + CUDA_PACKAGE=$(getLatestAzureLinuxNvidiaDriverPackageForKernel "cuda" "^cuda-[0-9]" "$KERNEL_VERSION") fi if [ -z "$CUDA_PACKAGE" ]; then diff --git a/pkg/agent/baker_test.go b/pkg/agent/baker_test.go index d54ef64979b..5614ea272b6 100644 --- a/pkg/agent/baker_test.go +++ b/pkg/agent/baker_test.go @@ -1081,6 +1081,10 @@ var _ = Describe("GetGPUDriverType", func() { It("should use cuda-lts with nc v3", func() { Expect(GetGPUDriverType("standard_nc6_v3")).To(Equal("cuda-lts")) }) + It("should use cuda-lts with GB", func() { + Expect(GetGPUDriverType("Standard_ND128isr_NDR_GB200_v6")).To(Equal("cuda-lts")) + Expect(GetGPUDriverType("Standard_ND128isr_GB300_v6")).To(Equal("cuda-lts")) + }) It("should keep cuda (legacy R470) with nc v1 (K80)", func() { Expect(GetGPUDriverType("standard_nc6")).To(Equal("cuda")) }) @@ -1119,6 +1123,13 @@ var _ = Describe("GetAKSGPUImageSHA", func() { }) }) +var _ = Describe("GPUNeedsFabricManager", func() { + It("should not use Fabric Manager with GB", func() { + Expect(GPUNeedsFabricManager("Standard_ND128isr_NDR_GB200_v6")).To(BeFalse()) + Expect(GPUNeedsFabricManager("Standard_ND128isr_GB300_v6")).To(BeFalse()) + }) +}) + var _ = Describe("getLinuxNodeCSECommand", func() { var ( templateGenerator *TemplateGenerator diff --git a/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh index cd06d3a6ea4..41be2e9c3ee 100755 --- a/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh @@ -2909,6 +2909,55 @@ OVERRIDE_EOF End End + Describe 'ARM64 GPU driver dispatch' + isARM64() { echo 1; } + logs_to_events() { + shift + "$@" + } + configGPUDrivers() { echo "configGPUDrivers called"; } + validateGPUDrivers() { echo "validateGPUDrivers called"; } + + Parameters + "$AZURELINUX_OS_NAME" "3.0" "" false true "configGPUDrivers called" + "$AZURELINUX_OS_NAME" "3.0" "" false false "validateGPUDrivers called" + "$UBUNTU_OS_NAME" "24.04" "" false true "" + "$AZURELINUX_OS_NAME" "2.0" "" false true "" + "$AZURELINUX_OS_NAME" "3.0" "$AZURELINUX_OSGUARD_OS_VARIANT" false true "" + "$AZURELINUX_OS_NAME" "3.0" "" true true "" + End + + It "dispatches ARM64 driver setup for OS=$1 version=$2 variant=$3 fips=$4 install=$5" + OS=$1 + OS_VERSION=$2 + OS_VARIANT=$3 + ENABLE_FIPS=$4 + CONFIG_GPU_DRIVER_IF_NEEDED=$5 + + When call ensureGPUDrivers + + The output should equal "$6" + End + End + + Describe 'ARM64 GPU driver validation' + isARM64() { echo 1; } + retrycmd_if_failure() { + echo "retrycmd_if_failure $*" >&2 + return 0 + } + which() { return 0; } + + It 'checks modprobe and nvidia-smi' + When call validateGPUDrivers + + The status should be success + The output should include "gpu driver loaded" + The stderr should include "retrycmd_if_failure 24 5 25 nvidia-modprobe -u -c0" + The stderr should include "retrycmd_if_failure 24 5 30 nvidia-smi" + End + End + Describe 'configGPUDrivers' # Assert the per-step CSE timing event names emitted via logs_to_events, # without running the real (hardware/daemon) driver steps. logs_to_events diff --git a/spec/parts/linux/cloud-init/artifacts/cse_install_mariner_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_install_mariner_spec.sh index 77cd8265788..e676b1375cd 100644 --- a/spec/parts/linux/cloud-init/artifacts/cse_install_mariner_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_install_mariner_spec.sh @@ -27,6 +27,7 @@ Describe 'cse_install_mariner.sh' BeforeAll 'setup' Include "./parts/linux/cloud-init/artifacts/cse_install.sh" Include "./parts/linux/cloud-init/artifacts/mariner/cse_install_mariner.sh" + Describe 'installDeps' It 'installs the required packages with installDeps for Mariner 2.0' OS_VERSION="2.0" @@ -340,6 +341,23 @@ Describe 'cse_install_mariner.sh' The variable GRID_CALLED should not equal "true" End + It 'selects the newest HWE OpenRM package for GB200' + NVIDIA_GPU_DRIVER_TYPE="cuda-lts" + MOCK_VM_SKU="Standard_ND128isr_NDR_GB200_v6" + MOCK_OPEN_RET=0 + uname() { echo "9.9.2-1.azl3"; } + dnf() { + echo "cuda-open-hwe-999.1.2-1_9.9.2.1.azl3.aarch64" + echo "cuda-open-hwe-999.1.2-2_9.9.2.1.azl3.aarch64" + echo "cuda-open-hwe-999.1.2-2_9.8.1.1.azl3.aarch64" + } + + When call downloadGPUDrivers + + The status should be success + The output should include "dnf install 30 1 600 cuda-open-hwe-999.1.2-2_9.9.2.1.azl3.aarch64" + End + It 'selects proprietary cuda path for T4 when NVIDIA_GPU_DRIVER_TYPE is cuda' NVIDIA_GPU_DRIVER_TYPE="cuda" MOCK_VM_SKU="Standard_NC4as_T4_v3" @@ -389,7 +407,7 @@ Describe 'cse_install_mariner.sh' dnf() { case "$4" in "cuda-open*") - echo "cuda-open-580.126.09-2_6.6.121.1.1.azl3.x86_64" + echo "cuda-open-hwe-999.1.2-2_6.6.121.1.1.azl3.x86_64" ;; "cuda") echo "cuda-570.195.03-1_6.6.121.1.1.azl3.x86_64" @@ -404,7 +422,7 @@ Describe 'cse_install_mariner.sh' The status should be success The output should include "NVIDIA GPU driver versions available at VHD build time for supported Azure Linux GPU VM sizes:" - The output should include " - nvidia-cuda-open-driver version 580.126.09" + The output should include " - nvidia-cuda-open-driver version 999.1.2" The output should include " - nvidia-cuda-driver version 570.195.03" The output should include " - nvidia-grid-driver version 570.211.01" The output should include "build-time snapshot only" From 22e49094187d0d12290148d9e07fe5991e1860c8 Mon Sep 17 00:00:00 2001 From: Mitch Zhu Date: Tue, 1 Sep 2026 18:27:18 +0000 Subject: [PATCH 3/3] test(vhd): validate shared Azure Linux ARM64 images Verify the dual-kernel image contract, GRUB selection, and ordinary ARM64 boot behavior. Run VHD validation from the requested fork and captured source SHA. Signed-off-by: Mitch Zhu --- .../linux_vhd_content_test_helpers_spec.sh | 18 +++- .../packer/test/linux-vhd-content-test.sh | 92 ++++++++++++++----- vhdbuilder/packer/test/run-test.sh | 3 +- 3 files changed, 87 insertions(+), 26 deletions(-) diff --git a/spec/vhdbuilder/packer/test/linux_vhd_content_test_helpers_spec.sh b/spec/vhdbuilder/packer/test/linux_vhd_content_test_helpers_spec.sh index 86b443782df..db022eb2f17 100644 --- a/spec/vhdbuilder/packer/test/linux_vhd_content_test_helpers_spec.sh +++ b/spec/vhdbuilder/packer/test/linux_vhd_content_test_helpers_spec.sh @@ -1,5 +1,5 @@ #!/bin/bash -# shellcheck disable=SC2329 +# shellcheck disable=SC2016,SC2329 # ShellSpec tests for parseAutologinSessions helper function @@ -233,3 +233,19 @@ Describe 'Inspektor Gadget version helper functions' End End End + +Describe 'AgentBaker source checkout' + SCRIPT='./vhdbuilder/packer/test/linux-vhd-content-test.sh' + + It 'downloads no-git archives from the configured repository' + When run grep -Fq 'AGENTBAKER_ARCHIVE_URL="${AGENTBAKER_REPOSITORY_URL%.git}/archive/${GIT_COMMIT_HASH}.tar.gz"' "$SCRIPT" + The status should be success + End + + It 'fetches the source ref and checks out the captured commit' + When run sed -n '/git fetch --quiet origin/,/git checkout --quiet/p' "$SCRIPT" + The output should include 'git fetch --quiet origin "$GIT_BRANCH"' + The output should include 'git checkout --quiet "$GIT_COMMIT_HASH"' + The status should be success + End +End diff --git a/vhdbuilder/packer/test/linux-vhd-content-test.sh b/vhdbuilder/packer/test/linux-vhd-content-test.sh index a3c4ee6c13d..91ab34d66fb 100644 --- a/vhdbuilder/packer/test/linux-vhd-content-test.sh +++ b/vhdbuilder/packer/test/linux-vhd-content-test.sh @@ -18,6 +18,7 @@ GIT_BRANCH="$4" IMG_SKU="$5" FEATURE_FLAGS="$6" GIT_COMMIT_HASH="$7" +AGENTBAKER_REPOSITORY_URL="${8:-https://github.com/Azure/AgentBaker.git}" systemctl daemon-reload && systemctl restart containerd @@ -62,15 +63,6 @@ assertPackageVersion() { return 0 } -# Clone the repo and checkout the branch provided. -# Simply clone with just the branch doesn't work for pull requests, but this technique works -# with everything we've tested so far. -# -# Strategy is to clone the repo, fetch the remote branch by ref into a local branch, and then checkout the local branch. -# The remote branch will be something like 'refs/heads/branch/name' or 'refs/pull/number/head'. Using the same name -# for the local branch has weird semantics, so we replace '/' with '-' for the local branch name. -LOCAL_GIT_BRANCH=${GIT_BRANCH//\//-} - SKIP_GIT_CLONE=false # Git is not present in the base image, so we need to install or bypass it. if [ "$OS_SKU" = "Ubuntu" ]; then @@ -90,8 +82,9 @@ if [ "$SKIP_GIT_CLONE" = "true" ]; then fi echo "Skipping git clone and pulling .tar.gz artifact for commit $GIT_COMMIT_HASH" - if ! curl -fLsS -o AgentBaker-${GIT_COMMIT_HASH}.tar.gz https://codeload.github.com/azure/agentbaker/tar.gz/${GIT_COMMIT_HASH}; then - err 'curl' "Failed to download https://codeload.github.com/azure/agentbaker/tar.gz/${GIT_COMMIT_HASH}" + AGENTBAKER_ARCHIVE_URL="${AGENTBAKER_REPOSITORY_URL%.git}/archive/${GIT_COMMIT_HASH}.tar.gz" + if ! curl -fLsS -o AgentBaker-${GIT_COMMIT_HASH}.tar.gz "$AGENTBAKER_ARCHIVE_URL"; then + err 'curl' "Failed to download $AGENTBAKER_ARCHIVE_URL" exit 1 fi if ! tar -xf AgentBaker-${GIT_COMMIT_HASH}.tar.gz; then @@ -100,12 +93,9 @@ if [ "$SKIP_GIT_CLONE" = "true" ]; then fi mv AgentBaker-${GIT_COMMIT_HASH} AgentBaker else - # Clone the AgentBaker repo and checkout the branch provided. - echo "Cloning AgentBaker repo and checking out remote branch '${GIT_BRANCH}' into local branch '${LOCAL_GIT_BRANCH}'" - COMMAND="git clone --quiet https://github.com/Azure/AgentBaker.git" - if ! ${COMMAND}; then + echo "Cloning configured AgentBaker repo and checking out commit '${GIT_COMMIT_HASH}' from '${GIT_BRANCH}'" + if ! git clone --quiet "$AGENTBAKER_REPOSITORY_URL" AgentBaker; then err 'git-clone' "Failed to clone AgentBaker repo" - err 'git-clone' "Used command '${COMMAND}'" exit 1 fi if ! pushd ./AgentBaker; then @@ -114,16 +104,12 @@ else err 'git-clone' "Contents of current directory: $(ls -al)" exit 1 fi - COMMAND="git fetch --quiet origin ${GIT_BRANCH}:${LOCAL_GIT_BRANCH}" - if ! ${COMMAND}; then - err 'git-clone' "Failed to fetch remote branch '${GIT_BRANCH}' into local branch '${LOCAL_GIT_BRANCH}'" - err 'git-clone' "Used command '${COMMAND}'" + if ! git fetch --quiet origin "$GIT_BRANCH"; then + err 'git-clone' "Failed to fetch AgentBaker ref '${GIT_BRANCH}'" exit 1 fi - COMMAND="git checkout --quiet ${LOCAL_GIT_BRANCH}" - if ! ${COMMAND}; then - err 'git-clone' "Failed to checkout local branch '${LOCAL_GIT_BRANCH}'" - err 'git-clone' "Used command '${COMMAND}'" + if ! git checkout --quiet "$GIT_COMMIT_HASH"; then + err 'git-clone' "Failed to checkout AgentBaker commit '${GIT_COMMIT_HASH}'" exit 1 fi if ! popd; then @@ -818,6 +804,63 @@ testLtsKernel() { } +testAzureLinuxArm64DualKernel() { + local test="testAzureLinuxArm64DualKernel" + local os_version=$1 + local os_sku=$2 + local enable_fips=$3 + + echo "$test:Start" + if [ "$os_sku" != "AzureLinux" ] || [ "$os_version" != "3.0" ] || [ "${enable_fips,,}" = "true" ] || [ "$(getCPUArch)" != "arm64" ]; then + echo "$test: Skipping for non-FIPS AzureLinux 3 ARM64 image" + return + fi + + local package + for package in kernel kernel-hwe grub2 grub2-efi-binary grub2-efi; do + if ! rpm -q "$package" >/dev/null 2>&1; then + err "$test" "$package is not installed" + fi + done + + local grub_version + local grub_efi_binary_version + local grub_efi_modules_version + grub_version=$(rpm -q --queryformat '%{VERSION}-%{RELEASE}' grub2 2>/dev/null || true) + grub_efi_binary_version=$(rpm -q --queryformat '%{VERSION}-%{RELEASE}' grub2-efi-binary 2>/dev/null || true) + grub_efi_modules_version=$(rpm -q --queryformat '%{VERSION}-%{RELEASE}' grub2-efi 2>/dev/null || true) + if [ -z "$grub_version" ] || [ "$grub_version" != "$grub_efi_binary_version" ] || [ "$grub_version" != "$grub_efi_modules_version" ]; then + err "$test" "GRUB package versions do not match: grub2=$grub_version, binary=$grub_efi_binary_version, modules=$grub_efi_modules_version" + fi + + local grub_module_file + for grub_module_file in extcmd.mod smbios.mod moddep.lst; do + if [ ! -s "/boot/grub2/arm64-efi/$grub_module_file" ]; then + err "$test" "/boot/grub2/arm64-efi/$grub_module_file is missing or empty" + fi + done + if ! grep -q '^smbios: extcmd$' /boot/grub2/arm64-efi/moddep.lst; then + err "$test" "GRUB smbios module dependency metadata is invalid" + fi + + if [ ! -x /etc/grub.d/10_azure_nvidia ] || [ ! -f /etc/default/grub.d/51-azure-nvidia.cfg ]; then + err "$test" "NVIDIA GRUB selector files are missing" + fi + if ! grub2-script-check /boot/grub2/grub.cfg; then + err "$test" "generated grub.cfg is invalid" + fi + if ! grep -q 'smbios --type 4 --get-string 7 --set cpu_manufacturer' /boot/grub2/grub.cfg; then + err "$test" "generated grub.cfg does not contain NVIDIA SMBIOS detection" + fi + + local running_kernel_package + running_kernel_package=$(rpm -qf --queryformat '%{NAME}' "/boot/vmlinuz-$(uname -r)" 2>/dev/null || true) + if [ "$running_kernel_package" != "kernel" ]; then + err "$test" "standard ARM64 test VM booted $running_kernel_package instead of kernel" + fi + echo "$test:Finish" +} + # Parse loginctl sessions to find console autologin sessions # Console autologin sessions have Remote=no and Service=login # Returns: Space-separated list of autologin session IDs @@ -2752,6 +2795,7 @@ testAKSNodeControllerBinary testAKSNodeControllerVersion testAKSNodeControllerService testLtsKernel $OS_VERSION $OS_SKU $ENABLE_FIPS +testAzureLinuxArm64DualKernel "$OS_VERSION" "$OS_SKU" "$ENABLE_FIPS" testAutologinDisabled $OS_SKU testCorednsBinaryExtractedAndCached $OS_VERSION checkLocaldnsScriptsAndConfigs $OS_SKU diff --git a/vhdbuilder/packer/test/run-test.sh b/vhdbuilder/packer/test/run-test.sh index 43a10c858f7..d94dd0217d0 100755 --- a/vhdbuilder/packer/test/run-test.sh +++ b/vhdbuilder/packer/test/run-test.sh @@ -141,13 +141,14 @@ if [ "${OS_TYPE,,}" = "linux" ]; then # If the pipeline that called this didn't set a branch, default to master. GIT_BRANCH="${GIT_BRANCH:-refs/heads/master}" GIT_COMMIT_HASH="${GIT_COMMIT_HASH:-$(git rev-parse HEAD)}" + AGENTBAKER_REPOSITORY_URL="${AGENTBAKER_REPOSITORY_URL:-https://github.com/Azure/AgentBaker.git}" SCRIPT_PATH="$CDIR/$LINUX_SCRIPT_PATH" for i in $(seq 1 3); do ret=$(az vm run-command invoke --command-id RunShellScript \ --name "$VM_NAME" \ --resource-group "$TEST_VM_RESOURCE_GROUP_NAME" \ --scripts "@$SCRIPT_PATH" \ - --parameters "${OS_VERSION}" "${ENABLE_FIPS}" "${OS_SKU}" "${GIT_BRANCH}" "${IMG_SKU}" "${FEATURE_FLAGS}" "${GIT_COMMIT_HASH}") && break + --parameters "${OS_VERSION}" "${ENABLE_FIPS}" "${OS_SKU}" "${GIT_BRANCH}" "${IMG_SKU}" "${FEATURE_FLAGS}" "${GIT_COMMIT_HASH}" "${AGENTBAKER_REPOSITORY_URL}") && break echo "${i}: retrying az vm run-command" done # The error message for a Linux VM run-command is as follows: