From 8772044d7321e1ee07e269bfe0b9b7a50fd9af14 Mon Sep 17 00:00:00 2001 From: chmill Date: Wed, 15 Jul 2026 02:00:28 +0000 Subject: [PATCH 01/38] feat: update snapshot to be generic --- CODEOWNERS | 3 + hotfix/hotfix_generate.py | 5 + .../artifacts/ubuntu/security-update.sh | 166 +++++++ .../ubuntu/ubuntu-snapshot-update.sh | 440 +++++++++++++----- parts/linux/cloud-init/nodecustomdata.yml | 22 + pkg/agent/const.go | 1 + pkg/agent/variables.go | 1 + .../artifacts/security-update_spec.sh | 185 ++++++++ .../artifacts/snapshot-update-service_spec.sh | 83 ++++ .../artifacts/ubuntu-snapshot-update_spec.sh | 418 +++++++++++++---- vhdbuilder/packer/packer_source.sh | 3 + .../packer/test/linux-vhd-content-test.sh | 32 ++ .../packer/vhd-image-builder-arm64-gb.json | 5 + .../packer/vhd-image-builder-arm64-gen2.json | 5 + vhdbuilder/packer/vhd-image-builder-base.json | 5 + vhdbuilder/packer/vhd-image-builder-cvm.json | 5 + 16 files changed, 1160 insertions(+), 219 deletions(-) create mode 100644 parts/linux/cloud-init/artifacts/ubuntu/security-update.sh create mode 100644 spec/parts/linux/cloud-init/artifacts/security-update_spec.sh create mode 100644 spec/parts/linux/cloud-init/artifacts/snapshot-update-service_spec.sh diff --git a/CODEOWNERS b/CODEOWNERS index 0809ee81744..fd151149cf4 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -17,7 +17,10 @@ parts/linux/cloud-init/artifacts/mariner/package-update.timer @yewmsft @YaoC parts/linux/cloud-init/artifacts/mariner/mariner-package-update.sh @yewmsft @YaoC parts/linux/cloud-init/artifacts/ubuntu/snapshot-update.service @yewmsft @YaoC parts/linux/cloud-init/artifacts/ubuntu/snapshot-update.timer @yewmsft @YaoC +parts/linux/cloud-init/artifacts/ubuntu/security-update.sh @yewmsft @YaoC parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh @yewmsft @YaoC +spec/parts/linux/cloud-init/artifacts/security-update_spec.sh @yewmsft @YaoC +spec/parts/linux/cloud-init/artifacts/snapshot-update-service_spec.sh @yewmsft @YaoC spec/parts/linux/cloud-init/artifacts/ubuntu-snapshot-update_spec.sh @yewmsft @YaoC spec/parts/linux/cloud-init/artifacts/mariner-package-update_spec.sh @yewmsft @YaoC diff --git a/hotfix/hotfix_generate.py b/hotfix/hotfix_generate.py index 48f58d0b72c..7cbd41877ef 100644 --- a/hotfix/hotfix_generate.py +++ b/hotfix/hotfix_generate.py @@ -86,7 +86,10 @@ "configure-azure-network.sh": "configureAzureNetworkScript", "init-aks-cloud.sh": "initAKSCloud", # Distro-specific scripts + # The updater and handler share one nodecustomdata block, so a change to + # either file hotfix-delivers both atomically. "ubuntu/ubuntu-snapshot-update.sh": "snapshotUpdateScript", + "ubuntu/security-update.sh": "securityUpdateScript", "mariner/mariner-package-update.sh": "packageUpdateScriptMariner", # Systemd services "kubelet.service": "kubeletSystemdService", @@ -97,6 +100,8 @@ "secure-tls-bootstrap.service": "secureTLSBootstrapService", "ensure-no-dup.service": "ensureNoDupEbtablesService", "measure-tls-bootstrapping-latency.service": "measureTLSBootstrappingLatencyService", + # These existing mappings remain part of the hotfix inventory, but the static + # units have no nodecustomdata write_files block and therefore are not injected. "ubuntu/snapshot-update.service": "snapshotUpdateService", "ubuntu/snapshot-update.timer": "snapshotUpdateTimer", "mariner/package-update.service": "packageUpdateServiceMariner", diff --git a/parts/linux/cloud-init/artifacts/ubuntu/security-update.sh b/parts/linux/cloud-init/artifacts/ubuntu/security-update.sh new file mode 100644 index 00000000000..49a4240406a --- /dev/null +++ b/parts/linux/cloud-init/artifacts/ubuntu/security-update.sh @@ -0,0 +1,166 @@ +#!/usr/bin/env bash + +# Applies the securityPatch component selected for this node's agent pool. This +# preserves the legacy Ubuntu snapshot-update behavior while moving desired state into the +# generic live-patching ConfigMap and status contract owned by the updater loop. + +: "${SECURITY_PATCH_CONFIG_DIR:=/var/lib/security-patch}" +: "${SECURITY_PATCH_DEFAULT_ENDPOINT:=snapshot.ubuntu.com}" + +security_patch_unattended_upgrade() { + local attempt + + for attempt in $(seq 1 10); do + if unattended-upgrade -v; then + echo "executed unattended upgrade ${attempt} times" + return 0 + fi + if [ "${attempt}" -lt 10 ]; then + sleep 5 + fi + done + + return 1 +} + +security_patch_generate_sources_list() { + local endpoint="$1" + local golden_timestamp="$2" + local code_name="$3" + + mkdir -p "${SECURITY_PATCH_CONFIG_DIR}" || return 1 + if [ "${endpoint}" = "${SECURITY_PATCH_DEFAULT_ENDPOINT}" ]; then + cat < "${SECURITY_PATCH_CONFIG_DIR}/sources.list" || return 1 +deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name} main restricted +deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-updates main restricted +deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name} universe +deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-updates universe +deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name} multiverse +deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-updates multiverse +deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-backports main restricted universe multiverse +deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-security main restricted +deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-security universe +deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-security multiverse +EOF + else + cat < "${SECURITY_PATCH_CONFIG_DIR}/sources.list" || return 1 +deb http://${endpoint}/ubuntu ${code_name} main restricted +deb http://${endpoint}/ubuntu ${code_name}-updates main restricted +deb http://${endpoint}/ubuntu ${code_name} universe +deb http://${endpoint}/ubuntu ${code_name}-updates universe +deb http://${endpoint}/ubuntu ${code_name} multiverse +deb http://${endpoint}/ubuntu ${code_name}-updates multiverse +deb http://${endpoint}/ubuntu ${code_name}-backports main restricted universe multiverse +deb http://${endpoint}/ubuntu ${code_name}-security main restricted +deb http://${endpoint}/ubuntu ${code_name}-security universe +deb http://${endpoint}/ubuntu ${code_name}-security multiverse +EOF + fi + + cat < "${SECURITY_PATCH_CONFIG_DIR}/apt.conf" || return 1 +Dir::Etc::sourcelist "${SECURITY_PATCH_CONFIG_DIR}/sources.list"; +Dir::Etc::sourceparts ""; +EOF +} + +security_patch_repo_endpoint() { + local node_json="$1" + local repo_service + local private_ip_regex='^((10\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})|(172\.(1[6-9]|2[0-9]|3[01])\.[0-9]{1,3}\.[0-9]{1,3})|(192\.168\.[0-9]{1,3}\.[0-9]{1,3}))$' + + if ! repo_service="$(printf '%s' "${node_json}" | jq -r '.metadata.annotations["kubernetes.azure.com/live-patching-repo-service"] // empty')"; then + echo "failed to read live patching repo service annotation" + return 1 + fi + + if [ -z "${repo_service}" ]; then + echo "${SECURITY_PATCH_DEFAULT_ENDPOINT}" + elif printf '%s' "${repo_service}" | grep -Eq "${private_ip_regex}"; then + echo "${repo_service}" + else + echo "ignoring invalid live patching repo service: ${repo_service}" >&2 + echo "${SECURITY_PATCH_DEFAULT_ENDPOINT}" + fi +} + +securityPatchIsCurrent() { + local desired_payload="$1" + local current_payload="$2" + local node_json="$3" + local agent_pool + local desired_timestamp + local current_timestamp + + if ! agent_pool="$(printf '%s' "${node_json}" | jq -er '.metadata.labels["kubernetes.azure.com/agentpool"] // empty')"; then + return 1 + fi + if ! desired_timestamp="$(printf '%s' "${desired_payload}" | jq -er --arg agentPool "${agent_pool}" '.agentPools[$agentPool].goldenTimestamp // empty')"; then + return 1 + fi + if ! current_timestamp="$(printf '%s' "${current_payload}" | jq -er --arg agentPool "${agent_pool}" '.agentPools[$agentPool].goldenTimestamp // empty')"; then + return 1 + fi + + [ "${desired_timestamp}" = "${current_timestamp}" ] +} + +updateSecurityPatch() { + local component_payload="${1:-}" + local node_json="${2:-}" + local agent_pool + local golden_timestamp + local repo_endpoint + local code_name + local apt_opts="-o Acquire::http::Timeout=300 -o Acquire::https::Timeout=300 -o Acquire::Retries=3" + + if [ -z "${node_json}" ]; then + echo "node JSON is required" + return 1 + fi + if ! agent_pool="$(printf '%s' "${node_json}" | jq -r '.metadata.labels["kubernetes.azure.com/agentpool"] // empty')"; then + echo "failed to read node agent pool label" + return 1 + fi + if [ -z "${agent_pool}" ]; then + echo "node agent pool label is not set" + return 1 + fi + + if ! golden_timestamp="$(printf '%s' "${component_payload}" | jq -er --arg agentPool "${agent_pool}" '.agentPools[$agentPool].goldenTimestamp // empty')"; then + echo "securityPatch profile is missing for agent pool: ${agent_pool}" + return 1 + fi + if ! printf '%s' "${golden_timestamp}" | grep -Eq '^[0-9]{8}T[0-9]{6}Z$'; then + echo "securityPatch goldenTimestamp is invalid: ${golden_timestamp}" + return 1 + fi + + if ! repo_endpoint="$(security_patch_repo_endpoint "${node_json}")"; then + return 1 + fi + if ! code_name="$(lsb_release -cs)" || [ -z "${code_name}" ]; then + echo "failed to determine Ubuntu codename" + return 1 + fi + if ! security_patch_generate_sources_list "${repo_endpoint}" "${golden_timestamp}" "${code_name}"; then + echo "failed to generate securityPatch apt configuration" + return 1 + fi + + export APT_CONFIG="${SECURITY_PATCH_CONFIG_DIR}/apt.conf" + if ! apt_get_update_with_opts "${apt_opts}"; then + echo "apt_get_update_with_opts failed" + return 1 + fi + if ! security_patch_unattended_upgrade; then + echo "unattended_upgrade failed" + return 1 + fi + + echo "securityPatch update completed successfully: ${golden_timestamp}" +} + +${__SOURCED__:+return} + +# shellcheck disable=SC1091 +source /opt/azure/containers/provision_source_distro.sh \ No newline at end of file diff --git a/parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh b/parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh index 6e200a8fb8b..e8545960baf 100755 --- a/parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh +++ b/parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh @@ -3,151 +3,357 @@ set -o nounset set -e -# Global constants used in this file. -# ------------------------------------------------------------------------------------------------- -SECURITY_PATCH_CONFIG_DIR=/var/lib/security-patch -KUBECONFIG="/var/lib/kubelet/kubeconfig" -KUBECTL="/opt/bin/kubectl --kubeconfig ${KUBECONFIG}" -DEFAULT_ENDPOINT="snapshot.ubuntu.com" - -# Function definitions used in this file. -# functions defined until "${__SOURCED__:+return}" are sourced and tested in - -# spec/parts/linux/cloud-init/artifacts/ubuntu-snapshot-update_spec.sh. -# ------------------------------------------------------------------------------------------------- -# Execute unattended-upgrade -unattended_upgrade() { - retries=10 - for i in $(seq 1 $retries); do - unattended-upgrade -v && break - if [ $i -eq $retries ]; then - return 1 - else sleep 5 - fi - done - echo Executed unattended upgrade $i times -} +# Runs the generic node-side component reconciliation loop. Component handlers own +# their payload schema and node mutations; this loop owns annotations, ConfigMap +# validation, dispatch, local component state, and final success reporting. -generate_sources_list() { - local endpoint="$1" - local golden_timestamp="$2" - local code_name="$3" - - mkdir -p "${SECURITY_PATCH_CONFIG_DIR}" - if [ "${endpoint}" = "${DEFAULT_ENDPOINT}" ]; then - cat << EOF > "${SECURITY_PATCH_CONFIG_DIR}/sources.list" -deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name} main restricted -deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-updates main restricted -deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name} universe -deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-updates universe -deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name} multiverse -deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-updates multiverse -deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-backports main restricted universe multiverse -deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-security main restricted -deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-security universe -deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-security multiverse -EOF - else - cat << EOF > "${SECURITY_PATCH_CONFIG_DIR}/sources.list" -deb http://${endpoint}/ubuntu ${code_name} main restricted -deb http://${endpoint}/ubuntu ${code_name}-updates main restricted -deb http://${endpoint}/ubuntu ${code_name} universe -deb http://${endpoint}/ubuntu ${code_name}-updates universe -deb http://${endpoint}/ubuntu ${code_name} multiverse -deb http://${endpoint}/ubuntu ${code_name}-updates multiverse -deb http://${endpoint}/ubuntu ${code_name}-backports main restricted universe multiverse -deb http://${endpoint}/ubuntu ${code_name}-security main restricted -deb http://${endpoint}/ubuntu ${code_name}-security universe -deb http://${endpoint}/ubuntu ${code_name}-security multiverse -EOF - fi - - cat << EOF > "${SECURITY_PATCH_CONFIG_DIR}/apt.conf" -Dir::Etc::sourcelist "${SECURITY_PATCH_CONFIG_DIR}/sources.list"; -Dir::Etc::sourceparts ""; -EOF - - echo "live patching configuration generated successfully" -} +: "${KUBECONFIG:=/var/lib/kubelet/kubeconfig}" +: "${KUBECTL:=/opt/bin/kubectl --kubeconfig ${KUBECONFIG}}" +: "${KNEAD_COMPONENT_CONFIG_NAMESPACE:=kube-system}" +: "${KNEAD_COMPONENT_CONFIGMAP:=live-patching-config}" +: "${KNEAD_COMPONENT_CONFIG_KEY_JSONPATH:=live-patching-config\.json}" +: "${KNEAD_COMPONENT_GOAL_ANNOTATION:=kubernetes.azure.com/live-patching-config-goal-hash}" +: "${KNEAD_COMPONENT_STATUS_ANNOTATION:=kubernetes.azure.com/live-patching-status}" +: "${KNEAD_COMPONENT_STATE_FILE:=/var/lib/aks/live-patching/current.json}" + +KNEAD_COMPONENT_RESULTS='{}' +KNEAD_COMPONENT_RESULTS_VALID=true -main() { - # At startup, we need to wait for kubelet to finish TLS bootstrapping to create the kubeconfig file. - while [ ! -f ${KUBECONFIG} ]; do - echo 'Waiting for TLS bootstrapping' +# Waits for kubelet credentials so kubectl can read Node and ConfigMap state. +knead_wait_for_kubeconfig() { + while [ ! -f "${KUBECONFIG}" ]; do + echo "waiting for kubelet kubeconfig" sleep 3 done +} + +# Reads this Node using the lowercase hostname expected from cloud provider registration. +knead_read_node() { + local node_name - node_name=$(hostname) + node_name="$(hostname)" if [ -z "${node_name}" ]; then - echo "cannot get node name" - exit 1 + echo "cannot get node name" >&2 + return 1 + fi + + node_name="$(printf '%s' "${node_name}" | tr '[:upper:]' '[:lower:]')" + + # shellcheck disable=SC2086 + $KUBECTL get node "${node_name}" -o json +} + +# Returns the value of the given annotation. +knead_get_node_annotation() { + local node_json="$1" + local annotation="$2" + + printf '%s' "${node_json}" | jq -r --arg annotation "${annotation}" '.metadata.annotations[$annotation] // empty' +} + +# Reads and validates the generic ConfigMap contract before any component handler runs. +# +# Knead validates the component envelope. Each handler owns its decoded +# nodeConfig schema. The goal must be the bare sha256 digest of the exact +# ConfigMap value read by this node. +knead_read_configmap() { + local goal="$1" + local payload + local payload_hash + + # shellcheck disable=SC2086 + if ! payload="$($KUBECTL get cm -n "${KNEAD_COMPONENT_CONFIG_NAMESPACE}" "${KNEAD_COMPONENT_CONFIGMAP}" -o "jsonpath={.data.${KNEAD_COMPONENT_CONFIG_KEY_JSONPATH}}")"; then + echo "failed to read live-patching-config ConfigMap" >&2 + return 1 + fi + + if ! printf '%s' "${payload}" | jq -e ' + (.components | type == "array") and + (.components | all( + (.name | type == "string") and + (.name | length > 0) and + (.nodeConfig | type == "string") + )) and + ([.components[].name] | length) == ([.components[].name] | unique | length) + ' > /dev/null; then + echo "live-patching-config payload has invalid envelope" >&2 + return 1 + fi + + if ! printf '%s' "${goal}" | grep -Eq '^[0-9a-f]{64}$'; then + echo "live-patching goal hash must be a 64-character lowercase sha256 digest" >&2 + return 1 + fi + + payload_hash="$(printf '%s' "${payload}" | sha256sum | awk '{print $1}')" + if [ "${payload_hash}" != "${goal}" ]; then + echo "live-patching goal hash does not match ConfigMap payload: goal=${goal}, payload=${payload_hash}" >&2 + return 1 fi - # Azure cloud provider assigns node name as the lowner case of the hostname - node_name=$(echo "$node_name" | tr '[:upper:]' '[:lower:]') + printf '%s' "${payload}" +} + +# Returns success when this exact component config was applied successfully but +# the overall Node status did not converge, such as when the annotation update or +# a sibling component failed. This local checkpoint prevents repeating successful +# work on the next retry. Missing or malformed state is treated as not current. +knead_component_is_current() { + local component="$1" + local component_payload="$2" + local node_json="$3" + local component_comparator="$4" + local current_payload + + if [ ! -f "${KNEAD_COMPONENT_STATE_FILE}" ]; then + return 1 + fi - # retrieve golden timestamp from node annotation - golden_timestamp=$($KUBECTL get node ${node_name} -o jsonpath="{.metadata.annotations['kubernetes\.azure\.com/live-patching-golden-timestamp']}") - if [ -z "${golden_timestamp}" ]; then - echo "golden timestamp is not set, skip live patching" - exit 0 + if ! current_payload="$(jq -er --arg component "${component}" \ + '.components | map(select(.name == $component)) | last | .nodeConfig' \ + "${KNEAD_COMPONENT_STATE_FILE}" 2> /dev/null)"; then + return 1 fi - echo "golden timestamp is: ${golden_timestamp}" - current_timestamp=$($KUBECTL get node ${node_name} -o jsonpath="{.metadata.annotations['kubernetes\.azure\.com/live-patching-current-timestamp']}") - if [ -n "${current_timestamp}" ]; then - echo "current timestamp is: ${current_timestamp}" + "${component_comparator}" "${component_payload}" "${current_payload}" "${node_json}" +} + +# Records one successfully applied component without changing sibling +# state. Missing or malformed state is rebuilt from an empty component list. +knead_write_component_state() { + local component="$1" + local component_payload="$2" + local state='{"components":[]}' + local state_dir + local state_tmp + local updated_at + + state_dir="$(dirname "${KNEAD_COMPONENT_STATE_FILE}")" + state_tmp="${KNEAD_COMPONENT_STATE_FILE}.tmp" + updated_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" - if [ "${golden_timestamp}" = "${current_timestamp}" ]; then - echo "golden and current timestamp is the same, nothing to patch" - exit 0 + if ! mkdir -p "${state_dir}"; then + echo "failed to create knead-component state directory" + return 1 + fi + if [ -f "${KNEAD_COMPONENT_STATE_FILE}" ]; then + if ! state="$(jq -c 'select(.components | type == "array") | {components: .components}' "${KNEAD_COMPONENT_STATE_FILE}" 2> /dev/null)" || [ -z "${state}" ]; then + state='{"components":[]}' fi fi + if ! printf '%s' "${state}" | jq \ + --arg component "${component}" \ + --arg componentPayload "${component_payload}" \ + --arg updatedAt "${updated_at}" \ + '.updatedAt = $updatedAt | .components = ([.components[] | select(.name != $component)] + [{name: $component, nodeConfig: $componentPayload}])' \ + > "${state_tmp}"; then + echo "failed to render knead-component state" + return 1 + fi + if ! mv "${state_tmp}" "${KNEAD_COMPONENT_STATE_FILE}"; then + echo "failed to write knead-component state" + return 1 + fi +} + +# Adds a component result to the status collection. +knead_set_component_result() { + local component="$1" + local code="$2" + local updated_results + + if ! updated_results="$(printf '%s' "${KNEAD_COMPONENT_RESULTS}" | jq -ce \ + --arg component "${component}" \ + --arg code "${code}" \ + '.[$component] = {code: $code}')" || [ -z "${updated_results}" ]; then + echo "failed to record component result: ${component}=${code}" + KNEAD_COMPONENT_RESULTS_VALID=false + return 1 + fi + + KNEAD_COMPONENT_RESULTS="${updated_results}" +} + +# Dispatches every known component and returns failure only after all have been attempted. +# +# This is the main error boundary for node disruption. A broken component should +# not stop unrelated handlers from making progress, so this function records +# failures and continues dispatching. Unknown components are skipped so older +# VHDs can still converge when a newer component appears in the shared config. +# +# Keep component payload parsing behind each handler. The generic loop should know +# only which handler owns a component name, not the shape of that component's JSON. +knead_apply_components() { + local payload="$1" + local node_json="$2" + local failed_components="" + local component + local component_count + local component_comparator + local component_handler + local component_index=0 + local component_payload + local infrastructure_failed=false - # Network isolated cluster can't access the internet, so we deploy a live patching repo service in the cluster - # The node will use the live patching repo service to download the repo metadata and packages - # If the annotation is not set, we will use the ubuntu snapshot repo - live_patching_repo_service=$($KUBECTL get node ${node_name} -o jsonpath="{.metadata.annotations['kubernetes\.azure\.com/live-patching-repo-service']}") - # Limit the live patching repo service to private IPs in the range of 10.x.x.x, 172.16.x.x - 172.31.x.x, and 192.168.x.x - private_ip_regex="^((10\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})|(172\.(1[6-9]|2[0-9]|3[01])\.[0-9]{1,3}\.[0-9]{1,3})|(192\.168\.[0-9]{1,3}\.[0-9]{1,3}))$" - # shellcheck disable=SC3010 - if [ -n "${live_patching_repo_service}" ] && [[ ! "${live_patching_repo_service}" =~ $private_ip_regex ]]; then - echo "Ignore invalid live patching repo service: ${live_patching_repo_service}" - live_patching_repo_service="" + KNEAD_COMPONENT_RESULTS='{}' + KNEAD_COMPONENT_RESULTS_VALID=true + + if ! component_count="$(printf '%s' "${payload}" | jq -er '.components | length')"; then + echo "failed to read component count" + KNEAD_COMPONENT_RESULTS_VALID=false + return 1 fi + while [ "${component_index}" -lt "${component_count}" ]; do + if ! component="$(printf '%s' "${payload}" | jq -er --argjson index "${component_index}" '.components[$index].name')"; then + echo "failed to read component name at index: ${component_index}" + infrastructure_failed=true + KNEAD_COMPONENT_RESULTS_VALID=false + component_index=$((component_index + 1)) + continue + fi + if ! component_payload="$(printf '%s' "${payload}" | jq -er --argjson index "${component_index}" '.components[$index].nodeConfig')"; then + echo "failed to read component payload: ${component}" + failed_components="${failed_components} ${component}" + infrastructure_failed=true + KNEAD_COMPONENT_RESULTS_VALID=false + if ! knead_set_component_result "${component}" "Failed"; then + infrastructure_failed=true + fi + component_index=$((component_index + 1)) + continue + fi + echo "applying component: ${component}" - repo_endpoint="${DEFAULT_ENDPOINT}" - if [ -z "${live_patching_repo_service}" ]; then - echo "live patching repo service is not set, use ubuntu snapshot repo" - else - echo "live patching repo service is: ${live_patching_repo_service}" - repo_endpoint="${live_patching_repo_service}" + # Select the handler for this component; unsupported components are ignored. + case "${component}" in + securityPatch) + component_comparator=securityPatchIsCurrent + component_handler=updateSecurityPatch + ;; + *) + echo "unsupported component: ${component}" + component_index=$((component_index + 1)) + continue + ;; + esac + + # Skip previously applied work; otherwise apply the component and checkpoint success. + if knead_component_is_current "${component}" "${component_payload}" "${node_json}" "${component_comparator}"; then + echo "component is already current: ${component}" + if ! knead_set_component_result "${component}" "Succeeded"; then + infrastructure_failed=true + fi + elif ! "${component_handler}" "${component_payload}" "${node_json}"; then + failed_components="${failed_components} ${component}" + echo "component failed: ${component}" + if ! knead_set_component_result "${component}" "Failed"; then + infrastructure_failed=true + fi + elif ! knead_write_component_state "${component}" "${component_payload}"; then + failed_components="${failed_components} ${component}" + echo "failed to persist component state: ${component}" + if ! knead_set_component_result "${component}" "Failed"; then + infrastructure_failed=true + fi + elif ! knead_set_component_result "${component}" "Succeeded"; then + infrastructure_failed=true + fi + component_index=$((component_index + 1)) + done + + if [ -n "${failed_components}" ]; then + echo "failed components:${failed_components}" + fi + if [ "${infrastructure_failed}" = true ]; then + echo "component dispatch encountered internal failures" + fi + [ -z "${failed_components}" ] && [ "${infrastructure_failed}" = false ] +} + +# Records the processed hash and per-component results in the node status annotation. +knead_write_status() { + local node_name="$1" + local goal="$2" + local status + + if [ "${KNEAD_COMPONENT_RESULTS_VALID}" != true ]; then + echo "refusing to write incomplete live-patching status" + return 1 + fi + if ! status="$(printf '%s' "${KNEAD_COMPONENT_RESULTS}" | jq -c --arg currentHash "${goal}" '{currentHash: $currentHash, components: .}')" || [ -z "${status}" ]; then + echo "failed to render live-patching status annotation" + return 1 fi - code_name=$(lsb_release -cs) - generate_sources_list "${repo_endpoint}" "${golden_timestamp}" "${code_name}" + # shellcheck disable=SC2086 + $KUBECTL annotate --overwrite node "${node_name}" "${KNEAD_COMPONENT_STATUS_ANNOTATION}=${status}" +} + +knead_main() { + local node_name + local node_json + local goal + local status + local payload + local result=0 - export APT_CONFIG="${SECURITY_PATCH_CONFIG_DIR}/apt.conf" + knead_wait_for_kubeconfig + if ! node_json="$(knead_read_node)"; then + echo "failed to read node" + return 1 + fi + if ! node_name="$(printf '%s' "${node_json}" | jq -er '.metadata.name')"; then + echo "failed to read node name" + return 1 + fi + if ! goal="$(knead_get_node_annotation "${node_json}" "${KNEAD_COMPONENT_GOAL_ANNOTATION}")"; then + echo "failed to read live-patching goal annotation" + return 1 + fi + if [ -z "${goal}" ]; then + echo "live-patching goal is not set, skip knead-component" + return 0 + fi + echo "live-patching goal is: ${goal}" - local apt_opts="-o Acquire::http::Timeout=300 -o Acquire::https::Timeout=300 -o Acquire::Retries=3" - if ! apt_get_update_with_opts "${apt_opts}"; then - echo "apt_get_update_with_opts failed" - exit 1 + if ! status="$(knead_get_node_annotation "${node_json}" "${KNEAD_COMPONENT_STATUS_ANNOTATION}")"; then + echo "failed to read live-patching status annotation" + return 1 fi - if ! unattended_upgrade; then - echo "unattended_upgrade failed" - exit 1 + # Skip reconciliation only when this goal was processed and every component succeeded. + if [ -n "${status}" ] && printf '%s' "${status}" | jq -e --arg goal "${goal}" \ + '.currentHash == $goal and (.components | type == "object") and (.components | all(.code == "Succeeded"))' \ + > /dev/null 2>&1; then + echo "live-patching goal is already converged, nothing to apply" + return 0 fi - # update current timestamp - $KUBECTL annotate --overwrite node ${node_name} kubernetes.azure.com/live-patching-current-timestamp=${golden_timestamp} + # Stop before dispatch when generic inputs are missing or unsafe. Once dispatch + # starts, knead_apply_components owns the continue-after-component-failure + # behavior so one broken handler does not prevent other handlers from running. + if ! payload="$(knead_read_configmap "${goal}")"; then + result=1 + else + if ! knead_apply_components "${payload}" "${node_json}"; then + result=1 + fi + + if ! knead_write_status "${node_name}" "${goal}"; then + echo "failed to update live-patching status annotation" + result=1 + fi + fi - echo snapshot update completed successfully + if [ "${result}" -eq 0 ]; then + echo "knead-component completed successfully" + fi + return "${result}" } ${__SOURCED__:+return} # --------------------------------------- Main Execution starts here -------------------------------------------------- -# source apt_get_update -source /opt/azure/containers/provision_source_distro.sh +# shellcheck disable=SC1091 +source /opt/azure/containers/security-update.sh -main "$@" +knead_main "$@" diff --git a/parts/linux/cloud-init/nodecustomdata.yml b/parts/linux/cloud-init/nodecustomdata.yml index 096f851953e..2d714536580 100644 --- a/parts/linux/cloud-init/nodecustomdata.yml +++ b/parts/linux/cloud-init/nodecustomdata.yml @@ -154,6 +154,28 @@ write_files: content: !!binary | {{GetVariableProperty "cloudInitData" "initAKSCloud"}} +# Deliver the generic reconciler and its security handler together so replacing +# the established updater path never leaves it without the dispatched handler. +{{if IsACL }} +{{- else if IsAzlOSGuard}} +{{- else if IsMariner}} +{{- else if IsFlatcar }} +{{- else }} +- path: /opt/azure/containers/ubuntu-snapshot-update.sh + permissions: "0544" + encoding: gzip + owner: root + content: !!binary | + {{GetVariableProperty "cloudInitData" "snapshotUpdateScript"}} + +- path: /opt/azure/containers/security-update.sh + permissions: "0544" + encoding: gzip + owner: root + content: !!binary | + {{GetVariableProperty "cloudInitData" "securityUpdateScript"}} +{{end}} + - path: /etc/systemd/system/reconcile-private-hosts.service permissions: "0644" encoding: gzip diff --git a/pkg/agent/const.go b/pkg/agent/const.go index ad5da5d2584..19d1043aa15 100644 --- a/pkg/agent/const.go +++ b/pkg/agent/const.go @@ -72,6 +72,7 @@ const ( bindMountScript = "linux/cloud-init/artifacts/bind-mount.sh" bindMountSystemdService = "linux/cloud-init/artifacts/bind-mount.service" snapshotUpdateScript = "linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh" + securityUpdateScript = "linux/cloud-init/artifacts/ubuntu/security-update.sh" snapshotUpdateSystemdService = "linux/cloud-init/artifacts/ubuntu/snapshot-update.service" snapshotUpdateSystemdTimer = "linux/cloud-init/artifacts/ubuntu/snapshot-update.timer" packageUpdateScriptMariner = "linux/cloud-init/artifacts/mariner/mariner-package-update.sh" diff --git a/pkg/agent/variables.go b/pkg/agent/variables.go index 545861b8198..54377c39e68 100644 --- a/pkg/agent/variables.go +++ b/pkg/agent/variables.go @@ -47,6 +47,7 @@ func getCustomDataVariables(config *datamodel.NodeBootstrappingConfiguration) pa "migPartitionScript": getBase64EncodedGzippedCustomScript(migPartitionScript, config), "ensureIMDSRestrictionScript": getBase64EncodedGzippedCustomScript(ensureIMDSRestrictionScript, config), "snapshotUpdateScript": getBase64EncodedGzippedCustomScript(snapshotUpdateScript, config), + "securityUpdateScript": getBase64EncodedGzippedCustomScript(securityUpdateScript, config), "snapshotUpdateService": getBase64EncodedGzippedCustomScript(snapshotUpdateSystemdService, config), "snapshotUpdateTimer": getBase64EncodedGzippedCustomScript(snapshotUpdateSystemdTimer, config), "packageUpdateScriptMariner": getBase64EncodedGzippedCustomScript(packageUpdateScriptMariner, config), diff --git a/spec/parts/linux/cloud-init/artifacts/security-update_spec.sh b/spec/parts/linux/cloud-init/artifacts/security-update_spec.sh new file mode 100644 index 00000000000..c6decd87140 --- /dev/null +++ b/spec/parts/linux/cloud-init/artifacts/security-update_spec.sh @@ -0,0 +1,185 @@ +#!/bin/bash +# shellcheck disable=SC2317 + +Describe 'security-update.sh' + security_patch_test_node_json() { + local repo_service="${1:-}" + + jq -nc --arg repoService "${repo_service}" \ + '{metadata: {labels: {"kubernetes.azure.com/agentpool": "ap1"}, annotations: {"kubernetes.azure.com/live-patching-repo-service": $repoService}}}' + } + + setup() { + Include ./parts/linux/cloud-init/artifacts/ubuntu/security-update.sh + TEST_DIR="/tmp/security-update-test" + rm -rf "${TEST_DIR}" + mkdir -p "${TEST_DIR}" + + SECURITY_PATCH_CONFIG_DIR="${TEST_DIR}/security-patch" + TEST_NODE_JSON="$(security_patch_test_node_json)" + TEST_APT_UPDATE_STATUS=0 + export SECURITY_PATCH_CONFIG_DIR TEST_NODE_JSON + export TEST_APT_UPDATE_STATUS + } + + cleanup() { + rm -rf "${TEST_DIR}" + } + + BeforeEach 'setup' + AfterEach 'cleanup' + + Mock lsb_release + echo "jammy" + End + + Mock unattended-upgrade + echo "unattended-upgrade called" + End + + Mock sleep + echo "sleep called" + End + + apt_get_update_with_opts() { + echo "apt_get_update_with_opts called with args: $*" + return "${TEST_APT_UPDATE_STATUS}" + } + + It 'applies the timestamp selected for the node agent pool' + When call updateSecurityPatch '{"agentPools":{"ap1":{"goldenTimestamp":"20260710T000000Z"},"ap2":{"goldenTimestamp":"20260701T000000Z"}}}' "${TEST_NODE_JSON}" + The status should be success + The output should include 'apt_get_update_with_opts called with args: -o Acquire::http::Timeout=300 -o Acquire::https::Timeout=300 -o Acquire::Retries=3' + The output should include 'unattended-upgrade called' + The output should include 'securityPatch update completed successfully: 20260710T000000Z' + The contents of file "${SECURITY_PATCH_CONFIG_DIR}/sources.list" should include 'deb https://snapshot.ubuntu.com/ubuntu/20260710T000000Z jammy main restricted' + The contents of file "${SECURITY_PATCH_CONFIG_DIR}/apt.conf" should include "Dir::Etc::sourcelist \"${SECURITY_PATCH_CONFIG_DIR}/sources.list\";" + End + + It 'compares only the current node agent pool timestamp' + desired_payload='{"agentPools":{"ap1":{"goldenTimestamp":"20260710T000000Z"},"ap2":{"goldenTimestamp":"20260715T000000Z"}}}' + current_payload='{"agentPools":{"ap1":{"goldenTimestamp":"20260710T000000Z"},"ap2":{"goldenTimestamp":"20260701T000000Z"}}}' + + When call securityPatchIsCurrent "${desired_payload}" "${current_payload}" "${TEST_NODE_JSON}" + The status should be success + End + + It 'detects a changed timestamp for the current node agent pool' + desired_payload='{"agentPools":{"ap1":{"goldenTimestamp":"20260710T000000Z"}}}' + current_payload='{"agentPools":{"ap1":{"goldenTimestamp":"20260701T000000Z"}}}' + + When call securityPatchIsCurrent "${desired_payload}" "${current_payload}" "${TEST_NODE_JSON}" + The status should be failure + End + + It 'retries unattended upgrade until it succeeds' + eval 'unattended-upgrade() { + TEST_UNATTENDED_ATTEMPT=$((TEST_UNATTENDED_ATTEMPT + 1)) + echo "unattended-upgrade called: ${TEST_UNATTENDED_ATTEMPT}" + [ "${TEST_UNATTENDED_ATTEMPT}" -ge 3 ] + }' + TEST_UNATTENDED_ATTEMPT=0 + export TEST_UNATTENDED_ATTEMPT + + When call security_patch_unattended_upgrade + The status should be success + The output should include 'unattended-upgrade called: 3' + The output should include 'executed unattended upgrade 3 times' + End + + It 'fails after ten unattended upgrade attempts' + eval 'unattended-upgrade() { + TEST_UNATTENDED_ATTEMPT=$((TEST_UNATTENDED_ATTEMPT + 1)) + echo "unattended-upgrade called: ${TEST_UNATTENDED_ATTEMPT}" + return 1 + }' + TEST_UNATTENDED_ATTEMPT=0 + export TEST_UNATTENDED_ATTEMPT + + When call security_patch_unattended_upgrade + The status should be failure + The output should include 'unattended-upgrade called: 10' + End + + It 'uses the private repository service for a network-isolated cluster' + TEST_NODE_JSON="$(security_patch_test_node_json '10.0.0.1')" + export TEST_NODE_JSON + + When call updateSecurityPatch '{"agentPools":{"ap1":{"goldenTimestamp":"20260710T000000Z"}}}' "${TEST_NODE_JSON}" + The status should be success + The output should include 'securityPatch update completed successfully: 20260710T000000Z' + The contents of file "${SECURITY_PATCH_CONFIG_DIR}/sources.list" should include 'deb http://10.0.0.1/ubuntu jammy main restricted' + The contents of file "${SECURITY_PATCH_CONFIG_DIR}/sources.list" should not include 'snapshot.ubuntu.com' + End + + It 'falls back to the snapshot service for an invalid repository annotation' + TEST_NODE_JSON="$(security_patch_test_node_json 'example.com')" + export TEST_NODE_JSON + + When call updateSecurityPatch '{"agentPools":{"ap1":{"goldenTimestamp":"20260710T000000Z"}}}' "${TEST_NODE_JSON}" + The status should be success + The output should include 'securityPatch update completed successfully: 20260710T000000Z' + The stderr should include 'ignoring invalid live patching repo service: example.com' + The contents of file "${SECURITY_PATCH_CONFIG_DIR}/sources.list" should include 'snapshot.ubuntu.com/ubuntu/20260710T000000Z' + End + + It 'fails when the component has no profile for the node agent pool' + When call updateSecurityPatch '{"agentPools":{"ap2":{"goldenTimestamp":"20260710T000000Z"}}}' "${TEST_NODE_JSON}" + The status should be failure + The output should include 'securityPatch profile is missing for agent pool: ap1' + The output should not include 'apt_get_update_with_opts called' + End + + It 'fails when the selected golden timestamp is malformed' + When call updateSecurityPatch '{"agentPools":{"ap1":{"goldenTimestamp":"not-a-timestamp"}}}' "${TEST_NODE_JSON}" + The status should be failure + The output should include 'securityPatch goldenTimestamp is invalid: not-a-timestamp' + The output should not include 'apt_get_update_with_opts called' + End + + It 'returns failure when apt metadata refresh fails' + TEST_APT_UPDATE_STATUS=1 + export TEST_APT_UPDATE_STATUS + + When call updateSecurityPatch '{"agentPools":{"ap1":{"goldenTimestamp":"20260710T000000Z"}}}' "${TEST_NODE_JSON}" + The status should be failure + The output should include 'apt_get_update_with_opts failed' + The output should not include 'unattended-upgrade called' + End + + It 'returns failure before apt update when apt configuration cannot be written' + SECURITY_PATCH_CONFIG_DIR="/dev/null/security-patch" + export SECURITY_PATCH_CONFIG_DIR + + When call updateSecurityPatch '{"agentPools":{"ap1":{"goldenTimestamp":"20260710T000000Z"}}}' "${TEST_NODE_JSON}" + The status should be failure + The output should include 'failed to generate securityPatch apt configuration' + The stderr should include 'Not a directory' + The output should not include 'apt_get_update_with_opts called' + The output should not include 'unattended-upgrade called' + End + + It 'does not mask a sources list write failure when apt config is writable' + mkdir -p "${SECURITY_PATCH_CONFIG_DIR}/sources.list" + + When call updateSecurityPatch '{"agentPools":{"ap1":{"goldenTimestamp":"20260710T000000Z"}}}' "${TEST_NODE_JSON}" + The status should be failure + The output should include 'failed to generate securityPatch apt configuration' + The stderr should include 'Is a directory' + The path "${SECURITY_PATCH_CONFIG_DIR}/apt.conf" should not be exist + The output should not include 'apt_get_update_with_opts called' + The output should not include 'unattended-upgrade called' + End + + It 'returns failure before apt update when the Ubuntu codename is unavailable' + Mock lsb_release + exit 1 + End + + When call updateSecurityPatch '{"agentPools":{"ap1":{"goldenTimestamp":"20260710T000000Z"}}}' "${TEST_NODE_JSON}" + The status should be failure + The output should include 'failed to determine Ubuntu codename' + The output should not include 'apt_get_update_with_opts called' + The output should not include 'unattended-upgrade called' + End +End \ No newline at end of file diff --git a/spec/parts/linux/cloud-init/artifacts/snapshot-update-service_spec.sh b/spec/parts/linux/cloud-init/artifacts/snapshot-update-service_spec.sh new file mode 100644 index 00000000000..f67923cb31c --- /dev/null +++ b/spec/parts/linux/cloud-init/artifacts/snapshot-update-service_spec.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +Describe 'snapshot update service commands' + validate_ubuntu_packer_inputs() { + local packer_file + + for packer_file in \ + vhdbuilder/packer/vhd-image-builder-base.json \ + vhdbuilder/packer/vhd-image-builder-cvm.json \ + vhdbuilder/packer/vhd-image-builder-arm64-gen2.json \ + vhdbuilder/packer/vhd-image-builder-arm64-gb.json + do + jq -e ' + [.. | objects | .source? // empty] as $sources | + ($sources | index("parts/linux/cloud-init/artifacts/ubuntu/security-update.sh")) != null and + ($sources | index("parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh")) != null and + ($sources | index("parts/linux/cloud-init/artifacts/ubuntu/snapshot-update.service")) != null and + ($sources | index("parts/linux/cloud-init/artifacts/ubuntu/snapshot-update.timer")) != null and + ($sources | index("parts/linux/cloud-init/artifacts/ubuntu/knead-component.sh")) == null + ' "${packer_file}" > /dev/null || return 1 + done + } + + validate_mariner_packer_inputs() { + local packer_file + + for packer_file in \ + vhdbuilder/packer/vhd-image-builder-mariner.json \ + vhdbuilder/packer/vhd-image-builder-mariner-cvm.json \ + vhdbuilder/packer/vhd-image-builder-mariner-arm64.json + do + jq -e ' + [.. | objects | .source? // empty] as $sources | + ($sources | index("parts/linux/cloud-init/artifacts/mariner/mariner-package-update.sh")) != null and + ($sources | index("parts/linux/cloud-init/artifacts/mariner/package-update.service")) != null and + ($sources | index("parts/linux/cloud-init/artifacts/mariner/package-update.timer")) != null + ' "${packer_file}" > /dev/null || return 1 + done + } + + validate_ubuntu_hotfix_transition() { + local template="parts/linux/cloud-init/nodecustomdata.yml" + local generator="hotfix/hotfix_generate.py" + local key + + for key in snapshotUpdateScript securityUpdateScript + do + grep -Fq "GetVariableProperty \"cloudInitData\" \"${key}\"" "${template}" || return 1 + grep -Fq ": \"${key}\"" "${generator}" || return 1 + done + + # Existing VHD-baked units remain unchanged; hotfixes update scripts only. + ! grep -Fq 'GetVariableProperty "cloudInitData" "snapshotUpdateService"' "${template}" || return 1 + ! grep -Fq 'GetVariableProperty "cloudInitData" "snapshotUpdateTimer"' "${template}" + } + + It 'keeps the Ubuntu snapshot service unchanged' + When run grep -Fx 'ExecStart=/opt/azure/containers/ubuntu-snapshot-update.sh' parts/linux/cloud-init/artifacts/ubuntu/snapshot-update.service + The status should be success + The output should equal 'ExecStart=/opt/azure/containers/ubuntu-snapshot-update.sh' + End + + It 'keeps the Azure Linux package updater' + When run grep -Fx 'ExecStart=/opt/azure/containers/mariner-package-update.sh' parts/linux/cloud-init/artifacts/mariner/package-update.service + The status should be success + The output should equal 'ExecStart=/opt/azure/containers/mariner-package-update.sh' + End + + It 'stages the generic updater, handler, and existing units for Ubuntu images' + When call validate_ubuntu_packer_inputs + The status should be success + End + + It 'preserves Azure Linux package updater inputs' + When call validate_mariner_packer_inputs + The status should be success + End + + It 'hotfix-delivers the generic updater and security handler without switching units' + When call validate_ubuntu_hotfix_transition + The status should be success + End +End \ No newline at end of file diff --git a/spec/parts/linux/cloud-init/artifacts/ubuntu-snapshot-update_spec.sh b/spec/parts/linux/cloud-init/artifacts/ubuntu-snapshot-update_spec.sh index 8f5cedcc5e2..aee0a77a846 100644 --- a/spec/parts/linux/cloud-init/artifacts/ubuntu-snapshot-update_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/ubuntu-snapshot-update_spec.sh @@ -1,15 +1,30 @@ #!/bin/bash +# shellcheck disable=SC2089,SC2090 -Describe 'ubuntu-snapshot-update.sh' +Describe 'ubuntu-snapshot-update.sh generic reconciliation' setup() { Include ./parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh - TEST_DIR="/tmp/live-patching-test" - mkdir -p ${TEST_DIR} - SECURITY_PATCH_CONFIG_DIR="${TEST_DIR}" + TEST_DIR="/tmp/knead-component-test" + rm -rf "${TEST_DIR}" + mkdir -p "${TEST_DIR}" + KUBECONFIG="${TEST_DIR}/kubeconfig" touch "${KUBECONFIG}" KUBECTL="kubectl" + KNEAD_COMPONENT_STATE_FILE="${TEST_DIR}/state/current.json" + TEST_COMPONENTS_JSON_FILE="${TEST_DIR}/components.json" + + TEST_STATUS="" + TEST_GOAL="" + TEST_AGENT_POOL="ap1" + TEST_REPO_SERVICE="" + printf '%s' '{"components":[]}' > "${TEST_COMPONENTS_JSON_FILE}" + TEST_SECURITY_STATUS=0 + TEST_ANNOTATE_STATUS=0 + export KUBECTL KNEAD_COMPONENT_STATE_FILE TEST_COMPONENTS_JSON_FILE + export TEST_STATUS TEST_GOAL TEST_AGENT_POOL TEST_REPO_SERVICE TEST_SECURITY_STATUS TEST_ANNOTATE_STATUS } + cleanup() { rm -rf "${TEST_DIR}" } @@ -17,108 +32,307 @@ Describe 'ubuntu-snapshot-update.sh' BeforeEach 'setup' AfterEach 'cleanup' - Mock apt_get_update_with_opts - echo "apt_get_update_with_opts mock called" - End - Mock unattended-upgrade - echo "unattended-upgrade mock called" - End - - It 'should update successfully for regular cluster' - Mock lsb_release - echo "jammy" - End - sources_list=$(cat < "${TEST_COMPONENTS_JSON_FILE}" + TEST_GOAL="$(sha256sum "${TEST_COMPONENTS_JSON_FILE}" | awk '{print $1}')" + export TEST_GOAL + } + + fail_result_then_write_status() { + KNEAD_COMPONENT_RESULTS='not-json' + KNEAD_COMPONENT_RESULTS_VALID=true + knead_set_component_result securityPatch Succeeded || true + echo "results valid: ${KNEAD_COMPONENT_RESULTS_VALID}" + knead_write_status aks-node-1 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + } + + fail_component_parse_then_write_status() { + knead_apply_components 'not-json' '{}' || true + echo "results valid: ${KNEAD_COMPONENT_RESULTS_VALID}" + knead_write_status aks-node-1 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + } + + It 'does nothing when goal annotation is not set' + TEST_GOAL="" + export TEST_GOAL + + When call knead_main + The status should be success + The output should include 'live-patching goal is not set, skip knead-component' + The output should not include 'updateSecurityPatch called' + The output should not include 'annotate mock called' + End + + It 'does nothing when status is converged for the goal' + TEST_GOAL="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + TEST_STATUS='{"currentHash":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","components":{"securityPatch":{"code":"Succeeded"}}}' + export TEST_GOAL TEST_STATUS + + When call knead_main + The status should be success + The output should include 'live-patching goal is already converged, nothing to apply' + The output should not include 'updateSecurityPatch called' + The output should not include 'annotate mock called' + End + + It 'dispatches securityPatch and writes successful status' + set_payload_goal '{"components":[{"name":"securityPatch","nodeConfig":"{\"agentPools\":{\"ap1\":{\"goldenTimestamp\":\"20260623T000000Z\"}}}"}]}' + + When call knead_main + The status should be success + The output should include 'applying component: securityPatch' + The output should include 'updateSecurityPatch called with args: {"agentPools":{"ap1":{"goldenTimestamp":"20260623T000000Z"}}}' + The output should include '"kubernetes.azure.com/agentpool":"ap1"' + The output should include 'annotate mock called with args: annotate --overwrite node aks-node-1 kubernetes.azure.com/live-patching-status={"currentHash":"' + The output should include '"components":{"securityPatch":{"code":"Succeeded"}}}' + The output should include 'knead-component completed successfully' + The contents of file "${KNEAD_COMPONENT_STATE_FILE}" should include '"securityPatch"' + End + + It 'fails before dispatch when the goal hash does not match the ConfigMap payload' + printf '%s' '{"components":[{"name":"securityPatch","nodeConfig":"{\"agentPools\":{}}"}]}' > "${TEST_COMPONENTS_JSON_FILE}" + TEST_GOAL="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + export TEST_GOAL + + When call knead_main + The status should be failure + The error should include 'live-patching goal hash does not match ConfigMap payload' + The output should not include 'updateSecurityPatch called' + The output should not include 'annotate mock called' + End + + It 'rejects a prefixed goal hash' + printf '%s' '{"components":[]}' > "${TEST_COMPONENTS_JSON_FILE}" + TEST_GOAL="sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + export TEST_GOAL + + When call knead_main + The status should be failure + The error should include 'live-patching goal hash must be a 64-character lowercase sha256 digest' + The output should not include 'annotate mock called' + End + + It 'fails before dispatch when components is not an array' + set_payload_goal '{"components":{}}' + + When call knead_main + The status should be failure + The error should include 'live-patching-config payload has invalid envelope' + The output should not include 'updateSecurityPatch called' + The output should not include 'annotate mock called' + End + + It 'fails before dispatch when component names are duplicated' + set_payload_goal '{"components":[{"name":"securityPatch","nodeConfig":"{\"agentPools\":{}}"},{"name":"securityPatch","nodeConfig":"{\"agentPools\":{}}"}]}' + + When call knead_main + The status should be failure + The error should include 'live-patching-config payload has invalid envelope' + The output should not include 'updateSecurityPatch called' + The output should not include 'annotate mock called' + End + + It 'records a failed securityPatch result and advances currentHash' + set_payload_goal '{"components":[{"name":"securityPatch","nodeConfig":"{\"agentPools\":{\"ap1\":{\"goldenTimestamp\":\"20260623T000000Z\"}}}"}]}' + TEST_SECURITY_STATUS=1 + export TEST_SECURITY_STATUS + + When call knead_main + The status should be failure + The output should include 'updateSecurityPatch called' + The output should include 'component failed: securityPatch' + The output should include 'failed components: securityPatch' + The output should include 'annotate mock called with args: annotate --overwrite node aks-node-1 kubernetes.azure.com/live-patching-status={"currentHash":"' + The output should include '"securityPatch":{"code":"Failed"}' + End + + It 'continues reporting later components after securityPatch fails' + set_payload_goal '{"components":[{"name":"securityPatch","nodeConfig":"{\"agentPools\":{\"ap1\":{\"goldenTimestamp\":\"20260623T000000Z\"}}}"},{"name":"npd","nodeConfig":"{\"version\":\"v20260623.0\"}"}]}' + TEST_SECURITY_STATUS=1 + export TEST_SECURITY_STATUS + + When call knead_main + The status should be failure + The output should include 'component failed: securityPatch' + The output should include 'unsupported component: npd' + The output should include 'failed components: securityPatch' + The output should include '"securityPatch":{"code":"Failed"}' + The output should not include '"npd"' + End + + It 'retries when currentHash matches but a component previously failed' + set_payload_goal '{"components":[{"name":"securityPatch","nodeConfig":"{\"agentPools\":{\"ap1\":{\"goldenTimestamp\":\"20260623T000000Z\"}}}"}]}' + TEST_STATUS="{\"currentHash\":\"${TEST_GOAL}\",\"components\":{\"securityPatch\":{\"code\":\"Failed\"}}}" + export TEST_STATUS + + When call knead_main The status should be success - The output should include 'live patching repo service is not set, use ubuntu snapshot repo' - The output should include 'apt_get_update_with_opts mock called' - The output should include 'unattended-upgrade mock called' - The output should include 'Executed unattended upgrade 1 times' - The output should include 'snapshot update completed successfully' - The contents of file "${SECURITY_PATCH_CONFIG_DIR}/sources.list" should eq "${sources_list}" - The contents of file "${SECURITY_PATCH_CONFIG_DIR}/apt.conf" should eq "${apt_config}" - End - - It 'should update successfully for ni cluster' - Mock lsb_release - echo "noble" - End - sources_list=$(cat < /dev/null || true + + When call knead_main The status should be success - The output should include 'golden timestamp is not set, skip live patching' + The output should include 'unsupported component: npd' + The output should include 'component is already current: securityPatch' + The output should not include 'updateSecurityPatch called' + The output should not include '"npd"' + The output should include '"securityPatch":{"code":"Succeeded"}' End - It 'should do nothing if golden timestamp equals current timestamp' - Mock kubectl - echo "20250820T000000Z" - End - When run main + It 'does not rerun an unchanged component when another component changes' + mkdir -p "$(dirname "${KNEAD_COMPONENT_STATE_FILE}")" + printf '%s' '{"components":[{"name":"securityPatch","nodeConfig":"{\"agentPools\":{\"ap1\":{\"goldenTimestamp\":\"20260623T000000Z\"}}}"}]}' > "${KNEAD_COMPONENT_STATE_FILE}" + set_payload_goal '{"components":[{"name":"npd","nodeConfig":"{\"version\":\"v20260701.0\"}"},{"name":"securityPatch","nodeConfig":"{\"agentPools\":{\"ap1\":{\"goldenTimestamp\":\"20260623T000000Z\"}}}"}]}' + + When call knead_main The status should be success - The output should include 'golden timestamp is: 20250820T000000Z' - The output should include 'current timestamp is: 20250820T000000Z' - The output should include 'golden and current timestamp is the same, nothing to patch' + The output should include 'unsupported component: npd' + The output should include 'component is already current: securityPatch' + The output should not include 'updateSecurityPatch called' + The output should not include '"npd"' + The output should include '"securityPatch":{"code":"Succeeded"}' + End + + It 'does not rerun securityPatch when only another agent pool changes' + mkdir -p "$(dirname "${KNEAD_COMPONENT_STATE_FILE}")" + printf '%s' '{"components":[{"name":"securityPatch","nodeConfig":"{\"agentPools\":{\"ap1\":{\"goldenTimestamp\":\"20260623T000000Z\"},\"ap2\":{\"goldenTimestamp\":\"20260623T000000Z\"}}}"}]}' > "${KNEAD_COMPONENT_STATE_FILE}" + set_payload_goal '{"components":[{"name":"securityPatch","nodeConfig":"{\"agentPools\":{\"ap1\":{\"goldenTimestamp\":\"20260623T000000Z\"},\"ap2\":{\"goldenTimestamp\":\"20260701T000000Z\"}}}"}]}' + + When call knead_main + The status should be success + The output should include 'component is already current: securityPatch' + The output should not include 'updateSecurityPatch called' + The output should include '"securityPatch":{"code":"Succeeded"}' + End + + It 'preserves sibling state when recording a successful component' + mkdir -p "$(dirname "${KNEAD_COMPONENT_STATE_FILE}")" + printf '%s' '{"components":[{"name":"npd","nodeConfig":"{\"version\":\"v20260623.0\"}"}]}' > "${KNEAD_COMPONENT_STATE_FILE}" + + When call knead_write_component_state securityPatch '{"agentPools":{"ap1":{"goldenTimestamp":"20260623T000000Z"}}}' + The status should be success + The contents of file "${KNEAD_COMPONENT_STATE_FILE}" should include '"npd"' + The contents of file "${KNEAD_COMPONENT_STATE_FILE}" should include 'v20260623.0' + The contents of file "${KNEAD_COMPONENT_STATE_FILE}" should include '"securityPatch"' + The contents of file "${KNEAD_COMPONENT_STATE_FILE}" should include '20260623T000000Z' + End + + It 'fails after handler success when status annotation update fails' + set_payload_goal '{"components":[{"name":"securityPatch","nodeConfig":"{\"agentPools\":{\"ap1\":{\"goldenTimestamp\":\"20260623T000000Z\"}}}"}]}' + TEST_ANNOTATE_STATUS=1 + export TEST_ANNOTATE_STATUS + + When call knead_main + The status should be failure + The output should include 'updateSecurityPatch called' + The output should include 'failed to update live-patching status annotation' + The output should not include 'knead-component completed successfully' + The contents of file "${KNEAD_COMPONENT_STATE_FILE}" should include '"securityPatch"' + End + + It 'fails before annotation when component results cannot be rendered' + KNEAD_COMPONENT_RESULTS='not-json' + export KNEAD_COMPONENT_RESULTS + + When call knead_write_status aks-node-1 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + The status should be failure + The output should include 'failed to render live-patching status annotation' + The stderr should include 'parse error' + The output should not include 'annotate mock called' + End + + It 'fails before annotation when status rendering produces no output' + KNEAD_COMPONENT_RESULTS='' + export KNEAD_COMPONENT_RESULTS + + When call knead_write_status aks-node-1 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + The status should be failure + The output should include 'failed to render live-patching status annotation' + The output should not include 'annotate mock called' + End + + It 'preserves existing results and refuses status after a result update fails' + When call fail_result_then_write_status + The status should be failure + The output should include 'failed to record component result: securityPatch=Succeeded' + The output should include 'results valid: false' + The output should include 'refusing to write incomplete live-patching status' + The stderr should include 'parse error' + The output should not include 'annotate mock called' + End + + It 'refuses status after component parsing fails' + When call fail_component_parse_then_write_status + The status should be failure + The output should include 'failed to read component count' + The output should include 'results valid: false' + The output should include 'refusing to write incomplete live-patching status' + The stderr should include 'parse error' + The output should not include 'annotate mock called' End -End +End \ No newline at end of file diff --git a/vhdbuilder/packer/packer_source.sh b/vhdbuilder/packer/packer_source.sh index 182c6332702..64f014a199b 100644 --- a/vhdbuilder/packer/packer_source.sh +++ b/vhdbuilder/packer/packer_source.sh @@ -216,6 +216,8 @@ copyPackerFiles() { KUBELET_SERVICE_DEST=/etc/systemd/system/kubelet.service SECURE_TLS_BOOTSTRAP_SERVICE_SRC=/home/packer/secure-tls-bootstrap.service SECURE_TLS_BOOTSTRAP_SERVICE_DEST=/etc/systemd/system/secure-tls-bootstrap.service + SECURITY_UPDATE_SH_SRC=/home/packer/security-update.sh + SECURITY_UPDATE_SH_DEST=/opt/azure/containers/security-update.sh USU_SH_SRC=/home/packer/ubuntu-snapshot-update.sh USU_SH_DEST=/opt/azure/containers/ubuntu-snapshot-update.sh MPU_SH_SRC=/home/packer/mariner-package-update.sh @@ -615,6 +617,7 @@ copyPackerFiles() { cpAndMode $PAM_D_COMMON_ACCOUNT_SRC $PAM_D_COMMON_ACCOUNT_DEST 644 cpAndMode $PAM_D_COMMON_AUTH_SRC $PAM_D_COMMON_AUTH_DEST 644 cpAndMode $PAM_D_COMMON_PASSWORD_SRC $PAM_D_COMMON_PASSWORD_DEST 644 + cpAndMode $SECURITY_UPDATE_SH_SRC $SECURITY_UPDATE_SH_DEST 544 cpAndMode $USU_SH_SRC $USU_SH_DEST 544 if [ "$UBUNTU_RELEASE" = "24.04" ] && [ "$CPU_ARCH" = "arm64" ]; then diff --git a/vhdbuilder/packer/test/linux-vhd-content-test.sh b/vhdbuilder/packer/test/linux-vhd-content-test.sh index f662230adb7..8e6b1d1d36a 100644 --- a/vhdbuilder/packer/test/linux-vhd-content-test.sh +++ b/vhdbuilder/packer/test/linux-vhd-content-test.sh @@ -2460,6 +2460,37 @@ checkLocaldnsScriptsAndConfigs() { #------------------------ End of test code related to localdns ------------------------ +testKneadSecurityPatchingAssets() { + local test="testKneadSecurityPatchingAssets" + local os_sku="$1" + local file + local permissions + local -A expected_files=( + ["/etc/systemd/system/snapshot-update.service"]=644 + ["/etc/systemd/system/snapshot-update.timer"]=644 + ["/opt/azure/containers/security-update.sh"]=544 + ["/opt/azure/containers/ubuntu-snapshot-update.sh"]=544 + ) + + if [ "$os_sku" != "Ubuntu" ]; then + return 0 + fi + + for file in "${!expected_files[@]}"; do + if [ ! -f "$file" ]; then + err "$test" "Expected file not found: $file" + fi + permissions=$(stat -c "%a" "$file") + if [ "$permissions" != "${expected_files[$file]}" ]; then + err "$test" "Incorrect permissions for $file. Expected ${expected_files[$file]}, got $permissions" + fi + done + + if ! grep -Fxq 'ExecStart=/opt/azure/containers/ubuntu-snapshot-update.sh' /etc/systemd/system/snapshot-update.service; then + err "$test" "snapshot-update.service does not execute the generic reconciler" + fi +} + # Basic sanity check for Inspektor Gadget artifacts baked into the image. testInspektorGadgetAssets() { local test="testInspektorGadgetAssets" @@ -2711,6 +2742,7 @@ testLtsKernel $OS_VERSION $OS_SKU $ENABLE_FIPS testAutologinDisabled $OS_SKU testCorednsBinaryExtractedAndCached $OS_VERSION checkLocaldnsScriptsAndConfigs $OS_SKU +testKneadSecurityPatchingAssets $OS_SKU testInspektorGadgetAssets testPackageDownloadURLFallbackLogic testFileOwnership $OS_SKU diff --git a/vhdbuilder/packer/vhd-image-builder-arm64-gb.json b/vhdbuilder/packer/vhd-image-builder-arm64-gb.json index 667be2946ae..80c9f5ae6ac 100644 --- a/vhdbuilder/packer/vhd-image-builder-arm64-gb.json +++ b/vhdbuilder/packer/vhd-image-builder-arm64-gb.json @@ -332,6 +332,11 @@ "source": "parts/linux/cloud-init/artifacts/setup-custom-search-domains.sh", "destination": "/home/packer/setup-custom-search-domains.sh" }, + { + "type": "file", + "source": "parts/linux/cloud-init/artifacts/ubuntu/security-update.sh", + "destination": "/home/packer/security-update.sh" + }, { "type": "file", "source": "parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh", diff --git a/vhdbuilder/packer/vhd-image-builder-arm64-gen2.json b/vhdbuilder/packer/vhd-image-builder-arm64-gen2.json index 8cc63a7241c..309b51d8369 100644 --- a/vhdbuilder/packer/vhd-image-builder-arm64-gen2.json +++ b/vhdbuilder/packer/vhd-image-builder-arm64-gen2.json @@ -317,6 +317,11 @@ "source": "parts/linux/cloud-init/artifacts/setup-custom-search-domains.sh", "destination": "/home/packer/setup-custom-search-domains.sh" }, + { + "type": "file", + "source": "parts/linux/cloud-init/artifacts/ubuntu/security-update.sh", + "destination": "/home/packer/security-update.sh" + }, { "type": "file", "source": "parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh", diff --git a/vhdbuilder/packer/vhd-image-builder-base.json b/vhdbuilder/packer/vhd-image-builder-base.json index a35a0807a18..09dcacccdea 100644 --- a/vhdbuilder/packer/vhd-image-builder-base.json +++ b/vhdbuilder/packer/vhd-image-builder-base.json @@ -320,6 +320,11 @@ "source": "parts/linux/cloud-init/artifacts/setup-custom-search-domains.sh", "destination": "/home/packer/setup-custom-search-domains.sh" }, + { + "type": "file", + "source": "parts/linux/cloud-init/artifacts/ubuntu/security-update.sh", + "destination": "/home/packer/security-update.sh" + }, { "type": "file", "source": "parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh", diff --git a/vhdbuilder/packer/vhd-image-builder-cvm.json b/vhdbuilder/packer/vhd-image-builder-cvm.json index f13b5ff46a0..d18ac41adb9 100644 --- a/vhdbuilder/packer/vhd-image-builder-cvm.json +++ b/vhdbuilder/packer/vhd-image-builder-cvm.json @@ -324,6 +324,11 @@ "source": "parts/linux/cloud-init/artifacts/setup-custom-search-domains.sh", "destination": "/home/packer/setup-custom-search-domains.sh" }, + { + "type": "file", + "source": "parts/linux/cloud-init/artifacts/ubuntu/security-update.sh", + "destination": "/home/packer/security-update.sh" + }, { "type": "file", "source": "parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh", From d0519a7967bf3be2119bc203bfa42973ffcd9c35 Mon Sep 17 00:00:00 2001 From: chmill Date: Wed, 15 Jul 2026 06:20:55 +0000 Subject: [PATCH 02/38] test: cover live patching ConfigMap JSONPath --- .../artifacts/ubuntu-snapshot-update_spec.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/spec/parts/linux/cloud-init/artifacts/ubuntu-snapshot-update_spec.sh b/spec/parts/linux/cloud-init/artifacts/ubuntu-snapshot-update_spec.sh index aee0a77a846..0916782a840 100644 --- a/spec/parts/linux/cloud-init/artifacts/ubuntu-snapshot-update_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/ubuntu-snapshot-update_spec.sh @@ -13,6 +13,7 @@ Describe 'ubuntu-snapshot-update.sh generic reconciliation' KUBECTL="kubectl" KNEAD_COMPONENT_STATE_FILE="${TEST_DIR}/state/current.json" TEST_COMPONENTS_JSON_FILE="${TEST_DIR}/components.json" + TEST_KUBECTL_ARGS_FILE="${TEST_DIR}/kubectl-args" TEST_STATUS="" TEST_GOAL="" @@ -21,7 +22,7 @@ Describe 'ubuntu-snapshot-update.sh generic reconciliation' printf '%s' '{"components":[]}' > "${TEST_COMPONENTS_JSON_FILE}" TEST_SECURITY_STATUS=0 TEST_ANNOTATE_STATUS=0 - export KUBECTL KNEAD_COMPONENT_STATE_FILE TEST_COMPONENTS_JSON_FILE + export KUBECTL KNEAD_COMPONENT_STATE_FILE TEST_COMPONENTS_JSON_FILE TEST_KUBECTL_ARGS_FILE export TEST_STATUS TEST_GOAL TEST_AGENT_POOL TEST_REPO_SERVICE TEST_SECURITY_STATUS TEST_ANNOTATE_STATUS } @@ -54,6 +55,7 @@ Describe 'ubuntu-snapshot-update.sh generic reconciliation' '{metadata: {name: $name, labels: {"kubernetes.azure.com/agentpool": $agentPool}, annotations: {"kubernetes.azure.com/live-patching-config-goal-hash": $goal, "kubernetes.azure.com/live-patching-status": $status, "kubernetes.azure.com/live-patching-repo-service": $repoService}}}' ;; *"get cm"*) + printf '%s' "$*" > "${TEST_KUBECTL_ARGS_FILE}" cat "${TEST_COMPONENTS_JSON_FILE}" ;; esac @@ -135,6 +137,15 @@ Describe 'ubuntu-snapshot-update.sh generic reconciliation' The contents of file "${KNEAD_COMPONENT_STATE_FILE}" should include '"securityPatch"' End + It 'reads the dotted ConfigMap key with an escaped JSONPath' + set_payload_goal '{"components":[]}' + + When call knead_read_configmap "${TEST_GOAL}" + The status should be success + The output should equal '{"components":[]}' + The contents of file "${TEST_KUBECTL_ARGS_FILE}" should equal 'get cm -n kube-system live-patching-config -o jsonpath={.data.live-patching-config\.json}' + End + It 'fails before dispatch when the goal hash does not match the ConfigMap payload' printf '%s' '{"components":[{"name":"securityPatch","nodeConfig":"{\"agentPools\":{}}"}]}' > "${TEST_COMPONENTS_JSON_FILE}" TEST_GOAL="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" From d27b056a0a90c840a7ed4d31c531a0be27d032e4 Mon Sep 17 00:00:00 2001 From: chmill Date: Wed, 22 Jul 2026 16:36:29 +0000 Subject: [PATCH 03/38] addressing comment, adding live patch annotate --- .../artifacts/ubuntu/security-update.sh | 16 +++++++++++ .../artifacts/security-update_spec.sh | 27 ++++++++++++++++--- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/parts/linux/cloud-init/artifacts/ubuntu/security-update.sh b/parts/linux/cloud-init/artifacts/ubuntu/security-update.sh index 49a4240406a..7b451e300f8 100644 --- a/parts/linux/cloud-init/artifacts/ubuntu/security-update.sh +++ b/parts/linux/cloud-init/artifacts/ubuntu/security-update.sh @@ -83,6 +83,8 @@ security_patch_repo_endpoint() { fi } +# Returns success when the desired security patch timestamp for this node's +# agent pool matches the timestamp in the locally checkpointed component state. securityPatchIsCurrent() { local desired_payload="$1" local current_payload="$2" @@ -104,11 +106,14 @@ securityPatchIsCurrent() { [ "${desired_timestamp}" = "${current_timestamp}" ] } +# Applies the security patch timestamp selected for this node's agent pool and, +# after patching succeeds, updates the legacy annotation consumed by the RP. updateSecurityPatch() { local component_payload="${1:-}" local node_json="${2:-}" local agent_pool local golden_timestamp + local node_name local repo_endpoint local code_name local apt_opts="-o Acquire::http::Timeout=300 -o Acquire::https::Timeout=300 -o Acquire::Retries=3" @@ -125,6 +130,10 @@ updateSecurityPatch() { echo "node agent pool label is not set" return 1 fi + if ! node_name="$(printf '%s' "${node_json}" | jq -er '.metadata.name // empty')"; then + echo "node name is not set" + return 1 + fi if ! golden_timestamp="$(printf '%s' "${component_payload}" | jq -er --arg agentPool "${agent_pool}" '.agentPools[$agentPool].goldenTimestamp // empty')"; then echo "securityPatch profile is missing for agent pool: ${agent_pool}" @@ -157,6 +166,13 @@ updateSecurityPatch() { return 1 fi + # Keep the legacy RP status channel current during migration to live-patching-status. + # shellcheck disable=SC2086 + if ! $KUBECTL annotate --overwrite node "${node_name}" "kubernetes.azure.com/live-patching-current-timestamp=${golden_timestamp}"; then + echo "failed to update legacy securityPatch status annotation" + return 1 + fi + echo "securityPatch update completed successfully: ${golden_timestamp}" } diff --git a/spec/parts/linux/cloud-init/artifacts/security-update_spec.sh b/spec/parts/linux/cloud-init/artifacts/security-update_spec.sh index c6decd87140..014f4f24337 100644 --- a/spec/parts/linux/cloud-init/artifacts/security-update_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/security-update_spec.sh @@ -6,7 +6,7 @@ Describe 'security-update.sh' local repo_service="${1:-}" jq -nc --arg repoService "${repo_service}" \ - '{metadata: {labels: {"kubernetes.azure.com/agentpool": "ap1"}, annotations: {"kubernetes.azure.com/live-patching-repo-service": $repoService}}}' + '{metadata: {name: "aks-node-1", labels: {"kubernetes.azure.com/agentpool": "ap1"}, annotations: {"kubernetes.azure.com/live-patching-repo-service": $repoService}}}' } setup() { @@ -18,8 +18,10 @@ Describe 'security-update.sh' SECURITY_PATCH_CONFIG_DIR="${TEST_DIR}/security-patch" TEST_NODE_JSON="$(security_patch_test_node_json)" TEST_APT_UPDATE_STATUS=0 - export SECURITY_PATCH_CONFIG_DIR TEST_NODE_JSON - export TEST_APT_UPDATE_STATUS + TEST_ANNOTATE_STATUS=0 + KUBECTL="kubectl" + export SECURITY_PATCH_CONFIG_DIR TEST_NODE_JSON KUBECTL + export TEST_APT_UPDATE_STATUS TEST_ANNOTATE_STATUS } cleanup() { @@ -41,6 +43,11 @@ Describe 'security-update.sh' echo "sleep called" End + Mock kubectl + echo "kubectl called with args: $*" + exit "${TEST_ANNOTATE_STATUS}" + End + apt_get_update_with_opts() { echo "apt_get_update_with_opts called with args: $*" return "${TEST_APT_UPDATE_STATUS}" @@ -51,6 +58,7 @@ Describe 'security-update.sh' The status should be success The output should include 'apt_get_update_with_opts called with args: -o Acquire::http::Timeout=300 -o Acquire::https::Timeout=300 -o Acquire::Retries=3' The output should include 'unattended-upgrade called' + The output should include 'kubectl called with args: annotate --overwrite node aks-node-1 kubernetes.azure.com/live-patching-current-timestamp=20260710T000000Z' The output should include 'securityPatch update completed successfully: 20260710T000000Z' The contents of file "${SECURITY_PATCH_CONFIG_DIR}/sources.list" should include 'deb https://snapshot.ubuntu.com/ubuntu/20260710T000000Z jammy main restricted' The contents of file "${SECURITY_PATCH_CONFIG_DIR}/apt.conf" should include "Dir::Etc::sourcelist \"${SECURITY_PATCH_CONFIG_DIR}/sources.list\";" @@ -145,6 +153,19 @@ Describe 'security-update.sh' The status should be failure The output should include 'apt_get_update_with_opts failed' The output should not include 'unattended-upgrade called' + The output should not include 'kubectl called' + End + + It 'returns failure when the legacy status annotation cannot be updated' + TEST_ANNOTATE_STATUS=1 + export TEST_ANNOTATE_STATUS + + When call updateSecurityPatch '{"agentPools":{"ap1":{"goldenTimestamp":"20260710T000000Z"}}}' "${TEST_NODE_JSON}" + The status should be failure + The output should include 'unattended-upgrade called' + The output should include 'kubectl called with args: annotate --overwrite node aks-node-1 kubernetes.azure.com/live-patching-current-timestamp=20260710T000000Z' + The output should include 'failed to update legacy securityPatch status annotation' + The output should not include 'securityPatch update completed successfully' End It 'returns failure before apt update when apt configuration cannot be written' From b36f5a79966b77fe16ba34722c9acdb8b62d9dbe Mon Sep 17 00:00:00 2001 From: sulixu Date: Wed, 5 Aug 2026 15:01:09 -0700 Subject: [PATCH 04/38] ci: extend GPU E2E timeout to three hours (#9141) Co-authored-by: sulixu <24964493+sulixu@users.noreply.github.com> --- .pipelines/e2e-gpu.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/e2e-gpu.yaml b/.pipelines/e2e-gpu.yaml index af36f7f6920..403e6298264 100644 --- a/.pipelines/e2e-gpu.yaml +++ b/.pipelines/e2e-gpu.yaml @@ -37,5 +37,5 @@ jobs: parameters: name: Ubuntu GPU Tests IgnoreScenariosWithMissingVhd: false - jobTimeoutInMinutes: 120 + jobTimeoutInMinutes: 180 failedTestsRetryCount: ${{ variables.E2E_FAILED_TESTS_RETRY_COUNT }} From 1906165049fe3fe15168b884ae7bf61a83a3cace Mon Sep 17 00:00:00 2001 From: aadhar-agarwal <108542189+aadhar-agarwal@users.noreply.github.com> Date: Wed, 5 Aug 2026 17:36:05 -0700 Subject: [PATCH 05/38] fix: prevent ACL kubelet sysext from starting before CSE (#9088) Signed-off-by: Aadhar Agarwal --- .../artifacts/acl/cse_install_acl.sh | 2 + .../linux/cloud-init/artifacts/cse_helpers.sh | 15 +++++++ .../artifacts/cse_install_acl_spec.sh | 43 +++++++++++++++++++ 3 files changed, 60 insertions(+) diff --git a/parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh b/parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh index c09c023d9a5..cdbb11d85e9 100644 --- a/parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh +++ b/parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh @@ -97,6 +97,8 @@ installCriCtlPackage() { } installKubeletKubectlFromPkg() { + maskKubeletSysextUpholds || exit $ERR_K8S_INSTALL_ERR + if mergeSysexts kubelet "${2:-mcr.microsoft.com}"/oss/v2/kubernetes/kubelet-sysext "$1" \ kubectl "${2:-mcr.microsoft.com}"/oss/v2/kubernetes/kubectl-sysext "$1"; then ln -snf /usr/bin/{kubelet,kubectl} /opt/bin/ diff --git a/parts/linux/cloud-init/artifacts/cse_helpers.sh b/parts/linux/cloud-init/artifacts/cse_helpers.sh index 863af3df0f1..6094ff4fa52 100755 --- a/parts/linux/cloud-init/artifacts/cse_helpers.sh +++ b/parts/linux/cloud-init/artifacts/cse_helpers.sh @@ -803,6 +803,21 @@ getSystemdArch() { esac } +maskKubeletSysextUpholds() { + local dropinDir="/etc/systemd/system/multi-user.target.d" + local dropinPath="${dropinDir}/10-kubelet-kubelet.conf" + + # AgentBaker owns kubelet activation, so suppress the sysext policy that starts it before CSE writes its configuration. + if ! mkdir -p "${dropinDir}"; then + echo "Failed to create kubelet sysext systemd drop-in directory ${dropinDir}" >&2 + return 1 + fi + if ! ln -sfn /dev/null "${dropinPath}"; then + echo "Failed to mask kubelet sysext systemd drop-in ${dropinPath}" >&2 + return 1 + fi +} + isARM64() { if [ "$(getCPUArch)" = "arm64" ]; then echo 1 diff --git a/spec/parts/linux/cloud-init/artifacts/cse_install_acl_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_install_acl_spec.sh index b63dbba6acf..76770aeb04c 100644 --- a/spec/parts/linux/cloud-init/artifacts/cse_install_acl_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_install_acl_spec.sh @@ -44,6 +44,49 @@ Describe 'cse_install_acl.sh' Include "./parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh" Include "./parts/linux/cloud-init/artifacts/cse_helpers.sh" + Describe 'installKubeletKubectlFromPkg' + It 'masks the kubelet sysext Upholds drop-in before activating the sysext' + MASK_CREATED=false + ln() { + if [ "$1" = "-sfn" ] && [ "$2" = "/dev/null" ] && [ "$3" = "/etc/systemd/system/multi-user.target.d/10-kubelet-kubelet.conf" ]; then + MASK_CREATED=true + fi + echo "mock ln $*" >&2 + } + mergeSysexts() { + if [ "$MASK_CREATED" != "true" ]; then + echo "mergeSysexts called before the kubelet sysext Upholds drop-in was masked" >&2 + return 1 + fi + echo "mock mergeSysexts $*" >&2 + } + When call installKubeletKubectlFromPkg "1.33" + The error should include "mock mkdir -p /etc/systemd/system/multi-user.target.d" + The error should include "mock ln -sfn /dev/null /etc/systemd/system/multi-user.target.d/10-kubelet-kubelet.conf" + The error should include "mock mergeSysexts kubelet mcr.microsoft.com/oss/v2/kubernetes/kubelet-sysext 1.33 kubectl mcr.microsoft.com/oss/v2/kubernetes/kubectl-sysext 1.33" + The error should include "mock ln -snf /usr/bin/kubelet /usr/bin/kubectl /opt/bin/" + The error should not include "mergeSysexts called before" + The status should be success + End + + It 'fails installation when the sysext Upholds drop-in cannot be masked' + mkdir() { + return 1 + } + mergeSysexts() { + echo "unexpected mergeSysexts" >&2 + } + installKubeletKubectlFromURL() { + echo "unexpected installKubeletKubectlFromURL" >&2 + } + When run installKubeletKubectlFromPkg "1.33" + The error should include "Failed to create kubelet sysext systemd drop-in directory /etc/systemd/system/multi-user.target.d" + The error should not include "unexpected mergeSysexts" + The error should not include "unexpected installKubeletKubectlFromURL" + The status should equal "$ERR_K8S_INSTALL_ERR" + End + End + Describe 'installSecureTLSBootstrapClientSysext' It 'calls mergeSysexts with correct URL and creates symlink on success' mergeSysexts() { From faf65021128d797f04f9613de337f858bc06d9c2 Mon Sep 17 00:00:00 2001 From: Ganeshkumar Ashokavardhanan <35557827+ganeshkumarashok@users.noreply.github.com> Date: Wed, 5 Aug 2026 18:55:31 -0700 Subject: [PATCH 06/38] fix(gpu): pull the GPU driver image from the cloud's own MCR endpoint (#9113) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 80cb6eea-a716-415c-ac0c-9ed20702761f --- .../linux/cloud-init/artifacts/cse_config.sh | 9 +++- .../linux/cloud-init/artifacts/cse_helpers.sh | 7 +++ .../cloud-init/artifacts/cse_helpers_spec.sh | 48 +++++++++++++++++++ 3 files changed, 63 insertions(+), 1 deletion(-) diff --git a/parts/linux/cloud-init/artifacts/cse_config.sh b/parts/linux/cloud-init/artifacts/cse_config.sh index 63a22927cc7..359933b84ce 100755 --- a/parts/linux/cloud-init/artifacts/cse_config.sh +++ b/parts/linux/cloud-init/artifacts/cse_config.sh @@ -1234,7 +1234,14 @@ pullGPUDriverImage() { # Cache-miss path only. Retry to ride out a transient blip, but stay tight: a truly missing image # should fail fast rather than eat the shared CSE window the driver install needs next. retrycmd # also self-caps to the CSE budget, so this can't overrun provisioning. - retrycmd_if_failure 3 5 120 ctr -n k8s.io image pull $NVIDIA_DRIVER_IMAGE:$NVIDIA_DRIVER_IMAGE_TAG + retrycmd_if_failure 3 5 120 ctr -n k8s.io image pull $NVIDIA_DRIVER_IMAGE_PULL_REF:$NVIDIA_DRIVER_IMAGE_TAG || return 1 + if [ "$NVIDIA_DRIVER_IMAGE_PULL_REF" != "$NVIDIA_DRIVER_IMAGE" ]; then + # Converge onto the canonical ref that install and cleanup use. Removing the pull ref drops + # only the name; the canonical tag still holds the manifest, so no blobs are collected. + ctr -n k8s.io image tag $NVIDIA_DRIVER_IMAGE_PULL_REF:$NVIDIA_DRIVER_IMAGE_TAG $NVIDIA_DRIVER_IMAGE:$NVIDIA_DRIVER_IMAGE_TAG || return 1 + ctr -n k8s.io image rm $NVIDIA_DRIVER_IMAGE_PULL_REF:$NVIDIA_DRIVER_IMAGE_TAG + fi + return 0 # the rm above is best effort; don't let it set the caller's exit code } installGPUDriverImage() { diff --git a/parts/linux/cloud-init/artifacts/cse_helpers.sh b/parts/linux/cloud-init/artifacts/cse_helpers.sh index 6094ff4fa52..39600643a91 100755 --- a/parts/linux/cloud-init/artifacts/cse_helpers.sh +++ b/parts/linux/cloud-init/artifacts/cse_helpers.sh @@ -199,7 +199,14 @@ export GPU_DEST=/usr/local/nvidia export NVIDIA_DRIVER_IMAGE_SHA="${GPU_IMAGE_SHA:=}" export NVIDIA_DRIVER_IMAGE_TAG="${GPU_DV}-${NVIDIA_DRIVER_IMAGE_SHA}" export NVIDIA_GPU_DRIVER_TYPE="${GPU_DRIVER_TYPE:=}" +# Canonical ref: the VHD bakes CUDA LTS under this exact name and configGPUDrivers matches it +# exactly, so it must stay mcr.microsoft.com in every cloud. export NVIDIA_DRIVER_IMAGE="mcr.microsoft.com/aks/aks-gpu-${NVIDIA_GPU_DRIVER_TYPE}" +# GRID is never baked, so it is pulled at provision time from this cloud's own MCR (sovereign clouds +# cannot reach mcr.microsoft.com). The base may carry a trailing slash and is unset during VHD build, +# where this file is sourced under `set -o nounset`, hence the default. +NVIDIA_DRIVER_IMAGE_MCR_BASE="${MCR_REPOSITORY_BASE:-mcr.microsoft.com}" +export NVIDIA_DRIVER_IMAGE_PULL_REF="${NVIDIA_DRIVER_IMAGE_MCR_BASE%/}/aks/aks-gpu-${NVIDIA_GPU_DRIVER_TYPE}" export CTR_GPU_INSTALL_CMD="ctr -n k8s.io run --privileged --rm --net-host --with-ns pid:/proc/1/ns/pid --mount type=bind,src=/opt/gpu,dst=/mnt/gpu,options=rbind --mount type=bind,src=/opt/actions,dst=/mnt/actions,options=rbind" export DOCKER_GPU_INSTALL_CMD="docker run --privileged --net=host --pid=host -v /opt/gpu:/mnt/gpu -v /opt/actions:/mnt/actions --rm" APT_CACHE_DIR=/var/cache/apt/archives/ diff --git a/spec/parts/linux/cloud-init/artifacts/cse_helpers_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_helpers_spec.sh index efb439c7db2..84a86d3c04e 100644 --- a/spec/parts/linux/cloud-init/artifacts/cse_helpers_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_helpers_spec.sh @@ -939,3 +939,51 @@ EOF End End End + +Describe 'GPU driver image reference resolution' + # Both references are computed when cse_helpers.sh is sourced, so each case re-sources the + # script in a clean child shell with the environment the RP would supply. + resolve_driver_refs() { + env -u NVIDIA_DRIVER_IMAGE -u NVIDIA_DRIVER_IMAGE_PULL_REF -u MCR_REPOSITORY_BASE "$@" bash -c \ + 'source ./parts/linux/cloud-init/artifacts/cse_helpers.sh >/dev/null 2>&1; echo "$NVIDIA_DRIVER_IMAGE $NVIDIA_DRIVER_IMAGE_PULL_REF"' + } + + It 'uses public MCR for both references when MCR_REPOSITORY_BASE is unset' + When call resolve_driver_refs GPU_DRIVER_TYPE=grid + The status should be success + The output should eq "mcr.microsoft.com/aks/aks-gpu-grid mcr.microsoft.com/aks/aks-gpu-grid" + End + + It 'uses public MCR for both references when MCR_REPOSITORY_BASE is empty' + When call resolve_driver_refs MCR_REPOSITORY_BASE= GPU_DRIVER_TYPE=grid + The status should be success + The output should eq "mcr.microsoft.com/aks/aks-gpu-grid mcr.microsoft.com/aks/aks-gpu-grid" + End + + It 'pulls from the sovereign MCR endpoint and strips the trailing slash' + When call resolve_driver_refs MCR_REPOSITORY_BASE=mcr.microsoft.scloud/ GPU_DRIVER_TYPE=grid + The status should be success + The output should eq "mcr.microsoft.com/aks/aks-gpu-grid mcr.microsoft.scloud/aks/aks-gpu-grid" + End + + It 'pulls from the sovereign MCR endpoint supplied without a trailing slash' + When call resolve_driver_refs MCR_REPOSITORY_BASE=mcr.microsoft.eaglex.ic.gov GPU_DRIVER_TYPE=grid + The status should be success + The output should eq "mcr.microsoft.com/aks/aks-gpu-grid mcr.microsoft.eaglex.ic.gov/aks/aks-gpu-grid" + End + + It 'applies the cloud-aware base to every driver type' + When call resolve_driver_refs MCR_REPOSITORY_BASE=mcr.azure.cn/ GPU_DRIVER_TYPE=cuda-lts + The status should be success + The output should eq "mcr.microsoft.com/aks/aks-gpu-cuda-lts mcr.azure.cn/aks/aks-gpu-cuda-lts" + End + + # Regression guard: the VHD bakes aks-gpu-cuda-lts under its mcr.microsoft.com name and + # configGPUDrivers matches it exactly, so a cloud-specific value here would break every + # sovereign CUDA node's cache hit. + It 'keeps the cache-facing reference on public MCR regardless of cloud' + When call resolve_driver_refs MCR_REPOSITORY_BASE=mcr.microsoft.scloud/ GPU_DRIVER_TYPE=cuda-lts + The status should be success + The output should start with "mcr.microsoft.com/aks/aks-gpu-cuda-lts " + End +End From 429e12617ec10cbb1612a79f0cd727fefdfcddc0 Mon Sep 17 00:00:00 2001 From: Tim Wright Date: Thu, 6 Aug 2026 16:40:26 +1200 Subject: [PATCH 07/38] fix: set containerd Windows service priority to ABOVE_NORMAL_PRIORITY_CLASS (#9146) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- e2e/scenario_win_test.go | 11 +++++++++++ e2e/validators.go | 21 +++++++++++++++++++++ staging/cse/windows/containerdfunc.ps1 | 1 + 3 files changed, 33 insertions(+) diff --git a/e2e/scenario_win_test.go b/e2e/scenario_win_test.go index 4eb18adb7f8..fe7064e2c3a 100644 --- a/e2e/scenario_win_test.go +++ b/e2e/scenario_win_test.go @@ -66,6 +66,7 @@ func Test_Windows2022_AzureNetwork(t *testing.T) { ValidateWindowsDisplayVersion(ctx, s, "21H2") ValidateFileHasContent(ctx, s, "/k/kubeletstart.ps1", "--container-runtime=remote") ValidateKubeletArgs(ctx, s) + ValidateContainerdWindowsPriorityClass(ctx, s) ValidateCiliumIsNotRunningWindows(ctx, s) ValidateDotnetNotInstalledWindows(ctx, s) ValidateWindowsSystemServicesRestartConfiguration(ctx, s) @@ -91,6 +92,7 @@ func Test_Windows2022AzureOverlayNetworkDualStack(t *testing.T) { ValidateWindowsDisplayVersion(ctx, s, "21H2") ValidateFileHasContent(ctx, s, "/k/kubeletstart.ps1", "--container-runtime=remote") ValidateKubeletArgs(ctx, s) + ValidateContainerdWindowsPriorityClass(ctx, s) ValidateCiliumIsNotRunningWindows(ctx, s) ValidateWindowsSystemServicesRestartConfiguration(ctx, s) ValidateCollectWindowsLogsScript(ctx, s) @@ -113,6 +115,7 @@ func Test_Windows2022Gen2AzureNetwork(t *testing.T) { ValidateWindowsDisplayVersion(ctx, s, "21H2") ValidateFileHasContent(ctx, s, "/k/kubeletstart.ps1", "--container-runtime=remote") ValidateKubeletArgs(ctx, s) + ValidateContainerdWindowsPriorityClass(ctx, s) ValidateCiliumIsNotRunningWindows(ctx, s) ValidateDotnetNotInstalledWindows(ctx, s) ValidateFileHasContent(ctx, s, "/AzureData/CustomDataSetupScript.log", "CSEScriptsPackageUrl used for provision is https://packages.aks.azure.com/aks/windows/cse/aks-windows-cse-scripts-current.zip") @@ -139,6 +142,7 @@ func Test_Windows2022Gen2AzureOverlayNetworkDualStack(t *testing.T) { ValidateWindowsDisplayVersion(ctx, s, "21H2") ValidateFileHasContent(ctx, s, "/k/kubeletstart.ps1", "--container-runtime=remote") ValidateKubeletArgs(ctx, s) + ValidateContainerdWindowsPriorityClass(ctx, s) ValidateCiliumIsNotRunningWindows(ctx, s) ValidateFileHasContent(ctx, s, "/AzureData/CustomDataSetupScript.log", "CSEScriptsPackageUrl used for provision is https://packages.aks.azure.com/aks/windows/cse/aks-windows-cse-scripts-current.zip") ValidateWindowsSystemServicesRestartConfiguration(ctx, s) @@ -183,6 +187,7 @@ func Test_Windows2025(t *testing.T) { ValidateWindowsDisplayVersion(ctx, s, "24H2") ValidateFileHasContent(ctx, s, "/k/kubeletstart.ps1", "--container-runtime=remote") ValidateKubeletArgs(ctx, s) + ValidateContainerdWindowsPriorityClass(ctx, s) ValidateCiliumIsNotRunningWindows(ctx, s) ValidateDotnetNotInstalledWindows(ctx, s) ValidateWindowsSystemServicesRestartConfiguration(ctx, s) @@ -208,6 +213,7 @@ func Test_Windows2025Gen2(t *testing.T) { ValidateWindowsDisplayVersion(ctx, s, "24H2") ValidateFileHasContent(ctx, s, "/k/kubeletstart.ps1", "--container-runtime=remote") ValidateKubeletArgs(ctx, s) + ValidateContainerdWindowsPriorityClass(ctx, s) ValidateCiliumIsNotRunningWindows(ctx, s) ValidateDotnetNotInstalledWindows(ctx, s) ValidateWindowsSystemServicesRestartConfiguration(ctx, s) @@ -235,6 +241,7 @@ func Test_Windows2025Gen2TrustedLaunch(t *testing.T) { ValidateWindowsDisplayVersion(ctx, s, "24H2") ValidateFileHasContent(ctx, s, "/k/kubeletstart.ps1", "--container-runtime=remote") ValidateKubeletArgs(ctx, s) + ValidateContainerdWindowsPriorityClass(ctx, s) ValidateCiliumIsNotRunningWindows(ctx, s) ValidateDotnetNotInstalledWindows(ctx, s) ValidateWindowsSystemServicesRestartConfiguration(ctx, s) @@ -266,6 +273,7 @@ func Test_Windows2025Gen2_WindowsCiliumNetworking(t *testing.T) { ValidateWindowsDisplayVersion(ctx, s, "24H2") ValidateFileHasContent(ctx, s, "/k/kubeletstart.ps1", "--container-runtime=remote") ValidateKubeletArgs(ctx, s) + ValidateContainerdWindowsPriorityClass(ctx, s) ValidateWindowsCiliumIsRunning(ctx, s) }, }, @@ -350,6 +358,7 @@ func Test_Windows2022_VHDCaching(t *testing.T) { ValidateWindowsDisplayVersion(ctx, s, "21H2") ValidateFileHasContent(ctx, s, "/k/kubeletstart.ps1", "--container-runtime=remote") ValidateKubeletArgs(ctx, s) + ValidateContainerdWindowsPriorityClass(ctx, s) ValidateCiliumIsNotRunningWindows(ctx, s) ValidateDotnetNotInstalledWindows(ctx, s) ValidateWindowsSystemServicesRestartConfiguration(ctx, s) @@ -378,6 +387,7 @@ func Test_Windows2025Gen2_VHDCaching(t *testing.T) { ValidateWindowsDisplayVersion(ctx, s, "24H2") ValidateFileHasContent(ctx, s, "/k/kubeletstart.ps1", "--container-runtime=remote") ValidateKubeletArgs(ctx, s) + ValidateContainerdWindowsPriorityClass(ctx, s) ValidateCiliumIsNotRunningWindows(ctx, s) ValidateDotnetNotInstalledWindows(ctx, s) ValidateWindowsSystemServicesRestartConfiguration(ctx, s) @@ -507,6 +517,7 @@ func Test_Windows2025Gen2_McrChinaCloud_Windows(t *testing.T) { ValidateWindowsDisplayVersion(ctx, s, "24H2") ValidateFileHasContent(ctx, s, "/k/kubeletstart.ps1", "--container-runtime=remote") ValidateKubeletArgs(ctx, s) + ValidateContainerdWindowsPriorityClass(ctx, s) ValidateCiliumIsNotRunningWindows(ctx, s) ValidateDotnetNotInstalledWindows(ctx, s) ValidateFileExists(ctx, s, `C:\ProgramData\containerd\certs.d\docker.io\hosts.toml`) diff --git a/e2e/validators.go b/e2e/validators.go index 3b8918c3d7c..4b8588a2481 100644 --- a/e2e/validators.go +++ b/e2e/validators.go @@ -1428,6 +1428,27 @@ func ValidateKubeletArgs(ctx context.Context, s *Scenario) { ValidateWindowsProcessHasCliArguments(ctx, s, "kubelet.exe", []string{"--rotate-certificates=true", "--client-ca-file=c:\\k\\ca.crt", "--windows-priorityclass=ABOVE_NORMAL_PRIORITY_CLASS"}) } +// ValidateContainerdWindowsPriorityClass verifies that the containerd service is registered +// with nssm's AppPriority set to ABOVE_NORMAL_PRIORITY_CLASS, and that the running containerd +// process actually has that OS process priority class applied. +func ValidateContainerdWindowsPriorityClass(ctx context.Context, s *Scenario) { + s.T.Helper() + + nssmCommand := strings.Join([]string{ + "$ErrorActionPreference = 'Stop'", + "& \"c:\\k\\nssm.exe\" get containerd AppPriority", + }, "\n") + nssmResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, nssmCommand, 0, "could not read containerd AppPriority from nssm") + require.Equal(s.T, "ABOVE_NORMAL_PRIORITY_CLASS", strings.TrimSpace(nssmResult.stdout), "expected containerd nssm service to be configured with AppPriority=ABOVE_NORMAL_PRIORITY_CLASS") + + processCommand := strings.Join([]string{ + "$ErrorActionPreference = 'Stop'", + "(Get-Process -Name containerd -ErrorAction Stop | Select-Object -First 1).PriorityClass", + }, "\n") + processResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, processCommand, 0, "could not read containerd process priority class") + require.Equal(s.T, "AboveNormal", strings.TrimSpace(processResult.stdout), "expected containerd process to be running with AboveNormal priority class") +} + func ValidateWindowsProcessHasCliArguments(ctx context.Context, s *Scenario, processName string, arguments []string) { steps := []string{ fmt.Sprintf("(Get-CimInstance Win32_Process -Filter \"name='%[1]s'\")[0].CommandLine", processName), diff --git a/staging/cse/windows/containerdfunc.ps1 b/staging/cse/windows/containerdfunc.ps1 index 79f132ff4b0..579c0dfab50 100644 --- a/staging/cse/windows/containerdfunc.ps1 +++ b/staging/cse/windows/containerdfunc.ps1 @@ -27,6 +27,7 @@ function RegisterContainerDService { & "$KubeDir\nssm.exe" set containerd Start SERVICE_DEMAND_START | RemoveNulls & "$KubeDir\nssm.exe" set containerd ObjectName LocalSystem | RemoveNulls & "$KubeDir\nssm.exe" set containerd Type SERVICE_WIN32_OWN_PROCESS | RemoveNulls + & "$KubeDir\nssm.exe" set containerd AppPriority ABOVE_NORMAL_PRIORITY_CLASS | RemoveNulls & "$KubeDir\nssm.exe" set containerd AppThrottle 1500 | RemoveNulls & "$KubeDir\nssm.exe" set containerd AppStdout "$KubeDir\containerd.log" | RemoveNulls & "$KubeDir\nssm.exe" set containerd AppStderr "$KubeDir\containerd.err.log" | RemoveNulls From fb9a9201bbceaeed56fed8c06294350319fa918a Mon Sep 17 00:00:00 2001 From: Nishchay Date: Thu, 6 Aug 2026 04:28:59 -0700 Subject: [PATCH 08/38] fix: revert "feat: add erofs snapshotter configuration for new kata-preview runtime class (#9085) (#9148) --- .../parser/templates/containerd.toml.gtpl | 22 ----- .../templates/containerd_no_GPU.toml.gtpl | 22 ----- .../parser/templates/containerd_v2.toml.gtpl | 21 ----- .../templates/containerd_v2_no_GPU.toml.gtpl | 21 ----- pkg/agent/baker.go | 86 ------------------- 5 files changed, 172 deletions(-) diff --git a/aks-node-controller/parser/templates/containerd.toml.gtpl b/aks-node-controller/parser/templates/containerd.toml.gtpl index ecd492bee80..6b9d7442fdb 100644 --- a/aks-node-controller/parser/templates/containerd.toml.gtpl +++ b/aks-node-controller/parser/templates/containerd.toml.gtpl @@ -1,27 +1,12 @@ version = 2 oom_score = -999{{if getHasDataDir .KubeletConfig}} root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} -{{- if .GetIsKata }} -[plugins."io.containerd.snapshotter.v1.erofs"] - default_size = "10G" - enable_fsverity = false - ovl_mount_options = [] - max_unmerged_layers = 1 - -[plugins."io.containerd.service.v1.diff-service"] - default = ["erofs", "walking"] - -[plugins."io.containerd.differ.v1.erofs"] - mkfs_options = ["-T0", "--mkfs-time", "--sort=none"] - enable_tar_index = false -{{- end}} [plugins."io.containerd.grpc.v1.cri"] sandbox_image = "{{ .KubeBinaryConfig.GetPodInfraContainerImageUrl }}" enable_cdi = true [plugins."io.containerd.grpc.v1.cri".containerd] {{- if .GetIsKata }} disable_snapshot_annotations = false - snapshotter = "overlayfs" {{- end}} {{- if .GetEnableArtifactStreaming }} snapshotter = "overlaybd" @@ -77,7 +62,6 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} {{- if .GetIsKata }} [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata] runtime_type = "io.containerd.kata.v2" - snapshotter = "overlayfs" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.katacli] runtime_type = "io.containerd.runc.v1" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.katacli.options] @@ -90,12 +74,6 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} Root = "" CriuPath = "" SystemdCgroup = false -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview] - runtime_type = "io.containerd.kata.v2" - privileged_without_host_devices = true - snapshotter = "erofs" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview.options] - ConfigPath = "/usr/share/defaults/kata-containers/configuration-clh-templating.toml" [proxy_plugins] [proxy_plugins.tardev] type = "snapshot" diff --git a/aks-node-controller/parser/templates/containerd_no_GPU.toml.gtpl b/aks-node-controller/parser/templates/containerd_no_GPU.toml.gtpl index 90c52b01120..2eb27cab54c 100644 --- a/aks-node-controller/parser/templates/containerd_no_GPU.toml.gtpl +++ b/aks-node-controller/parser/templates/containerd_no_GPU.toml.gtpl @@ -1,26 +1,11 @@ version = 2 oom_score = -999{{if getHasDataDir .KubeletConfig}} root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} -{{- if .GetIsKata }} -[plugins."io.containerd.snapshotter.v1.erofs"] - default_size = "10G" - enable_fsverity = false - ovl_mount_options = [] - max_unmerged_layers = 1 - -[plugins."io.containerd.service.v1.diff-service"] - default = ["erofs", "walking"] - -[plugins."io.containerd.differ.v1.erofs"] - mkfs_options = ["-T0", "--mkfs-time", "--sort=none"] - enable_tar_index = false -{{- end}} [plugins."io.containerd.grpc.v1.cri"] sandbox_image = "{{ .KubeBinaryConfig.GetPodInfraContainerImageUrl }}" [plugins."io.containerd.grpc.v1.cri".containerd] {{- if .GetIsKata }} disable_snapshot_annotations = false - snapshotter = "overlayfs" {{- end}} {{- if .GetEnableArtifactStreaming }} snapshotter = "overlaybd" @@ -61,7 +46,6 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} {{- if .GetIsKata }} [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata] runtime_type = "io.containerd.kata.v2" - snapshotter = "overlayfs" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.katacli] runtime_type = "io.containerd.runc.v1" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.katacli.options] @@ -74,12 +58,6 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} Root = "" CriuPath = "" SystemdCgroup = false -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview] - runtime_type = "io.containerd.kata.v2" - privileged_without_host_devices = true - snapshotter = "erofs" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview.options] - ConfigPath = "/usr/share/defaults/kata-containers/configuration-clh-templating.toml" [proxy_plugins] [proxy_plugins.tardev] type = "snapshot" diff --git a/aks-node-controller/parser/templates/containerd_v2.toml.gtpl b/aks-node-controller/parser/templates/containerd_v2.toml.gtpl index 42442b8c57f..fd11c508145 100644 --- a/aks-node-controller/parser/templates/containerd_v2.toml.gtpl +++ b/aks-node-controller/parser/templates/containerd_v2.toml.gtpl @@ -1,20 +1,6 @@ version = 2 oom_score = -999{{if getHasDataDir .KubeletConfig}} root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} -{{- if .GetIsKata }} -[plugins."io.containerd.snapshotter.v1.erofs"] - default_size = "10G" - enable_fsverity = false - ovl_mount_options = [] - max_unmerged_layers = 1 - -[plugins."io.containerd.service.v1.diff-service"] - default = ["erofs", "walking"] - -[plugins."io.containerd.differ.v1.erofs"] - mkfs_options = ["-T0", "--mkfs-time", "--sort=none"] - enable_tar_index = false -{{- end}} [plugins."io.containerd.cri.v1.images"] {{- if .GetEnableArtifactStreaming }} snapshotter = "overlaybd" @@ -70,15 +56,8 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata] runtime_type = "io.containerd.kata.v2" privileged_without_host_devices = true - snapshotter = "overlayfs" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options] ConfigPath = "/usr/share/defaults/kata-containers/configuration.toml" -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview] - runtime_type = "io.containerd.kata.v2" - privileged_without_host_devices = true - snapshotter = "erofs" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview.options] - ConfigPath = "/usr/share/defaults/kata-containers/configuration-clh-templating.toml" [proxy_plugins] [proxy_plugins.tardev] type = "snapshot" diff --git a/aks-node-controller/parser/templates/containerd_v2_no_GPU.toml.gtpl b/aks-node-controller/parser/templates/containerd_v2_no_GPU.toml.gtpl index c28dd4f560c..45a2129f743 100644 --- a/aks-node-controller/parser/templates/containerd_v2_no_GPU.toml.gtpl +++ b/aks-node-controller/parser/templates/containerd_v2_no_GPU.toml.gtpl @@ -1,20 +1,6 @@ version = 2 oom_score = -999{{if getHasDataDir .KubeletConfig}} root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} -{{- if .GetIsKata }} -[plugins."io.containerd.snapshotter.v1.erofs"] - default_size = "10G" - enable_fsverity = false - ovl_mount_options = [] - max_unmerged_layers = 1 - -[plugins."io.containerd.service.v1.diff-service"] - default = ["erofs", "walking"] - -[plugins."io.containerd.differ.v1.erofs"] - mkfs_options = ["-T0", "--mkfs-time", "--sort=none"] - enable_tar_index = false -{{- end}} [plugins."io.containerd.cri.v1.images"] {{- if .GetEnableArtifactStreaming }} snapshotter = "overlaybd" @@ -57,15 +43,8 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata] runtime_type = "io.containerd.kata.v2" privileged_without_host_devices = true - snapshotter = "overlayfs" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options] ConfigPath = "/usr/share/defaults/kata-containers/configuration.toml" -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview] - runtime_type = "io.containerd.kata.v2" - privileged_without_host_devices = true - snapshotter = "erofs" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview.options] - ConfigPath = "/usr/share/defaults/kata-containers/configuration-clh-templating.toml" [proxy_plugins] [proxy_plugins.tardev] type = "snapshot" diff --git a/pkg/agent/baker.go b/pkg/agent/baker.go index 6fbb0a4ef86..14f3b79d851 100644 --- a/pkg/agent/baker.go +++ b/pkg/agent/baker.go @@ -1890,27 +1890,12 @@ const ( containerdV1ConfigTemplate ContainerdConfigTemplate = `version = 2 oom_score = -999{{if HasDataDir }} root = "{{GetDataDir}}"{{- end}} -{{- if IsKata }} -[plugins."io.containerd.snapshotter.v1.erofs"] - default_size = "10G" - enable_fsverity = false - ovl_mount_options = [] - max_unmerged_layers = 1 - -[plugins."io.containerd.service.v1.diff-service"] - default = ["erofs", "walking"] - -[plugins."io.containerd.differ.v1.erofs"] - mkfs_options = ["-T0", "--mkfs-time", "--sort=none"] - enable_tar_index = false -{{- end}} [plugins."io.containerd.grpc.v1.cri"] sandbox_image = "{{GetPodInfraContainerSpec}}" enable_cdi = true [plugins."io.containerd.grpc.v1.cri".containerd] {{- if IsKata }} disable_snapshot_annotations = false - snapshotter = "overlayfs" {{- end}} {{- if IsArtifactStreamingEnabled }} snapshotter = "overlaybd" @@ -1967,15 +1952,8 @@ root = "{{GetDataDir}}"{{- end}} [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata] runtime_type = "io.containerd.kata.v2" privileged_without_host_devices = true - snapshotter = "overlayfs" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options] ConfigPath = "/usr/share/defaults/kata-containers/configuration.toml" -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview] - runtime_type = "io.containerd.kata.v2" - privileged_without_host_devices = true - snapshotter = "erofs" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview.options] - ConfigPath = "/usr/share/defaults/kata-containers/configuration-clh-templating.toml" [proxy_plugins] [proxy_plugins.tardev] type = "snapshot" @@ -1992,20 +1970,6 @@ root = "{{GetDataDir}}"{{- end}} containerdV2ConfigTemplate ContainerdConfigTemplate = `version = 2 oom_score = -999{{if HasDataDir }} root = "{{GetDataDir}}"{{- end}} -{{- if IsKata }} -[plugins."io.containerd.snapshotter.v1.erofs"] - default_size = "10G" - enable_fsverity = false - ovl_mount_options = [] - max_unmerged_layers = 1 - -[plugins."io.containerd.service.v1.diff-service"] - default = ["erofs", "walking"] - -[plugins."io.containerd.differ.v1.erofs"] - mkfs_options = ["-T0", "--mkfs-time", "--sort=none"] - enable_tar_index = false -{{- end}} [plugins."io.containerd.cri.v1.images"] {{- if IsArtifactStreamingEnabled }} snapshotter = "overlaybd" @@ -2061,15 +2025,8 @@ root = "{{GetDataDir}}"{{- end}} [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata] runtime_type = "io.containerd.kata.v2" privileged_without_host_devices = true - snapshotter = "overlayfs" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options] ConfigPath = "/usr/share/defaults/kata-containers/configuration.toml" -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview] - runtime_type = "io.containerd.kata.v2" - privileged_without_host_devices = true - snapshotter = "erofs" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview.options] - ConfigPath = "/usr/share/defaults/kata-containers/configuration-clh-templating.toml" [proxy_plugins] [proxy_plugins.tardev] type = "snapshot" @@ -2086,20 +2043,6 @@ root = "{{GetDataDir}}"{{- end}} containerdV2NoGPUConfigTemplate ContainerdConfigTemplate = `version = 2 oom_score = -999{{if HasDataDir }} root = "{{GetDataDir}}"{{- end}} -{{- if IsKata }} -[plugins."io.containerd.snapshotter.v1.erofs"] - default_size = "10G" - enable_fsverity = false - ovl_mount_options = [] - max_unmerged_layers = 1 - -[plugins."io.containerd.service.v1.diff-service"] - default = ["erofs", "walking"] - -[plugins."io.containerd.differ.v1.erofs"] - mkfs_options = ["-T0", "--mkfs-time", "--sort=none"] - enable_tar_index = false -{{- end}} [plugins."io.containerd.cri.v1.images"] {{- if IsArtifactStreamingEnabled }} snapshotter = "overlaybd" @@ -2142,15 +2085,8 @@ root = "{{GetDataDir}}"{{- end}} [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata] runtime_type = "io.containerd.kata.v2" privileged_without_host_devices = true - snapshotter = "overlayfs" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options] ConfigPath = "/usr/share/defaults/kata-containers/configuration.toml" -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview] - runtime_type = "io.containerd.kata.v2" - privileged_without_host_devices = true - snapshotter = "erofs" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview.options] - ConfigPath = "/usr/share/defaults/kata-containers/configuration-clh-templating.toml" [proxy_plugins] [proxy_plugins.tardev] type = "snapshot" @@ -2160,26 +2096,11 @@ root = "{{GetDataDir}}"{{- end}} containerdV1NoGPUConfigTemplate ContainerdConfigTemplate = `version = 2 oom_score = -999{{if HasDataDir }} root = "{{GetDataDir}}"{{- end}} -{{- if IsKata }} -[plugins."io.containerd.snapshotter.v1.erofs"] - default_size = "10G" - enable_fsverity = false - ovl_mount_options = [] - max_unmerged_layers = 1 - -[plugins."io.containerd.service.v1.diff-service"] - default = ["erofs", "walking"] - -[plugins."io.containerd.differ.v1.erofs"] - mkfs_options = ["-T0", "--mkfs-time", "--sort=none"] - enable_tar_index = false -{{- end}} [plugins."io.containerd.grpc.v1.cri"] sandbox_image = "{{GetPodInfraContainerSpec}}" [plugins."io.containerd.grpc.v1.cri".containerd] {{- if IsKata }} disable_snapshot_annotations = false - snapshotter = "overlayfs" {{- end}} {{- if IsArtifactStreamingEnabled }} snapshotter = "overlaybd" @@ -2221,15 +2142,8 @@ root = "{{GetDataDir}}"{{- end}} [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata] runtime_type = "io.containerd.kata.v2" privileged_without_host_devices = true - snapshotter = "overlayfs" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options] ConfigPath = "/usr/share/defaults/kata-containers/configuration.toml" -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview] - runtime_type = "io.containerd.kata.v2" - privileged_without_host_devices = true - snapshotter = "erofs" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview.options] - ConfigPath = "/usr/share/defaults/kata-containers/configuration-clh-templating.toml" [proxy_plugins] [proxy_plugins.tardev] type = "snapshot" From 9969884eaa953884b590eae03e4d8edd91041918 Mon Sep 17 00:00:00 2001 From: Sylvain Boily <4981802+djsly@users.noreply.github.com> Date: Thu, 6 Aug 2026 07:38:14 -0400 Subject: [PATCH 09/38] fix: persist Azure Linux THP settings after reboot (#9136) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: af53ff70-d493-484f-9865-074752cd22aa --- e2e/exec.go | 38 +- e2e/node_config.go | 8 +- e2e/scenario_test.go | 70 ++++ e2e/test_helpers.go | 7 +- e2e/validators.go | 98 ++++- e2e/vmss.go | 2 +- .../linux/cloud-init/artifacts/cse_config.sh | 255 ++++++++++++- parts/linux/cloud-init/artifacts/cse_main.sh | 9 + pkg/agent/baker.go | 45 ++- pkg/agent/bakerapi.go | 4 +- pkg/agent/bakerapi_test.go | 48 +++ pkg/agent/utils_test.go | 74 +++- .../cloud-init/artifacts/cse_config_spec.sh | 361 ++++++++++++++++++ .../cse_main_disable_modules_spec.sh | 11 + 14 files changed, 979 insertions(+), 51 deletions(-) diff --git a/e2e/exec.go b/e2e/exec.go index 260e3c8f627..7ca9c23b987 100644 --- a/e2e/exec.go +++ b/e2e/exec.go @@ -8,7 +8,6 @@ import ( "path/filepath" "strconv" "strings" - "sync" "time" scp "github.com/bramvdbogaerde/go-scp" @@ -19,12 +18,6 @@ import ( "k8s.io/client-go/tools/remotecommand" ) -var bufferPool = sync.Pool{ - New: func() any { - return new(bytes.Buffer) - }, -} - type podExecResult struct { exitCode string stderr, stdout string @@ -111,17 +104,26 @@ func runSSHCommandWithPrivateKeyFile( } defer session.Close() - stdout := bufferPool.Get().(*bytes.Buffer) - stderr := bufferPool.Get().(*bytes.Buffer) - stdout.Reset() - stderr.Reset() - - defer bufferPool.Put(stdout) - defer bufferPool.Put(stderr) - session.Stdout = stdout - session.Stderr = stderr - - err = session.Run(command) + var stdout bytes.Buffer + var stderr bytes.Buffer + session.Stdout = &stdout + session.Stderr = &stderr + + runErr := make(chan error, 1) + go func() { + runErr <- session.Run(command) + }() + select { + case err = <-runErr: + case <-ctx.Done(): + _ = session.Close() + _ = client.Close() + select { + case <-runErr: + case <-time.After(5 * time.Second): + } + return nil, fmt.Errorf("SSH command canceled or timed out: %w", ctx.Err()) + } exitCode := 0 if err != nil { diff --git a/e2e/node_config.go b/e2e/node_config.go index 6e161116b39..1bd82d24244 100644 --- a/e2e/node_config.go +++ b/e2e/node_config.go @@ -142,9 +142,11 @@ func getBaseNBC(ctx context.Context, t testing.TB, cluster *Cluster, vhd *config // is a temporary workaround // eventually we want to phase out usage of nbc -func nbcToAKSNodeConfigV1(nbc *datamodel.NodeBootstrappingConfiguration) *aksnodeconfigv1.Configuration { +func nbcToAKSNodeConfigV1(nbc *datamodel.NodeBootstrappingConfiguration) (*aksnodeconfigv1.Configuration, error) { cs := nbc.ContainerService - agent.ValidateAndSetLinuxNodeBootstrappingConfiguration(nbc) + if err := agent.ValidateAndSetLinuxNodeBootstrappingConfigurationWithError(nbc); err != nil { + return nil, err + } bootstrappingConfig := &aksnodeconfigv1.BootstrappingConfig{ TlsBootstrappingToken: nbc.KubeletClientTLSBootstrapToken, @@ -377,7 +379,7 @@ func nbcToAKSNodeConfigV1(nbc *datamodel.NodeBootstrappingConfiguration) *aksnod cfg.KubeletConfig.KubeletFlags = kubeletFlags } - return cfg + return cfg, nil } // this is huge, but accurate, so leave it here. diff --git a/e2e/scenario_test.go b/e2e/scenario_test.go index dc6fc03302f..f1b660dc4f6 100644 --- a/e2e/scenario_test.go +++ b/e2e/scenario_test.go @@ -1631,6 +1631,76 @@ func Test_AzureLinuxV3_CustomSysctls(t *testing.T) { }) } +func Test_AzureLinuxV3_CustomLinuxOSConfigPersistsAfterReboot(t *testing.T) { + customSysctls := map[string]string{ + "net.ipv4.ip_local_port_range": "32768 62535", + "net.netfilter.nf_conntrack_max": "2097152", + "net.netfilter.nf_conntrack_buckets": "524288", + "net.ipv4.tcp_keepalive_intvl": "90", + } + customContainerdUlimits := map[string]string{ + "LimitMEMLOCK": "75000", + "LimitNOFILE": "1048", + } + const ( + swapFileSizeMB int32 = 64 + thpEnabled = "never" + thpDefrag = "never" + ) + + RunScenario(t, &Scenario{ + Description: "tests that AzureLinuxV3 custom Linux OS config persists after a node reboot", + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDAzureLinuxV3Gen2, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.AgentPoolProfile.CustomLinuxOSConfig = &datamodel.CustomLinuxOSConfig{ + Sysctls: &datamodel.SysctlConfig{ + NetNetfilterNfConntrackMax: to.Ptr(toolkit.StrToInt32(customSysctls["net.netfilter.nf_conntrack_max"])), + NetNetfilterNfConntrackBuckets: to.Ptr(toolkit.StrToInt32(customSysctls["net.netfilter.nf_conntrack_buckets"])), + NetIpv4IpLocalPortRange: customSysctls["net.ipv4.ip_local_port_range"], + NetIpv4TcpkeepaliveIntvl: to.Ptr(toolkit.StrToInt32(customSysctls["net.ipv4.tcp_keepalive_intvl"])), + }, + UlimitConfig: &datamodel.UlimitConfig{ + MaxLockedMemory: customContainerdUlimits["LimitMEMLOCK"], + NoFile: customContainerdUlimits["LimitNOFILE"], + }, + SwapFileSizeMB: to.Ptr(swapFileSizeMB), + TransparentHugePageEnabled: thpEnabled, + TransparentHugePageDefrag: thpDefrag, + } + nbc.AgentPoolProfile.CustomKubeletConfig = &datamodel.CustomKubeletConfig{ + FailSwapOn: to.Ptr(false), + } + }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) { + config.CustomLinuxOsConfig = &aksnodeconfigv1.CustomLinuxOsConfig{ + SysctlConfig: &aksnodeconfigv1.SysctlConfig{ + NetNetfilterNfConntrackMax: to.Ptr(toolkit.StrToInt32(customSysctls["net.netfilter.nf_conntrack_max"])), + NetNetfilterNfConntrackBuckets: to.Ptr(toolkit.StrToInt32(customSysctls["net.netfilter.nf_conntrack_buckets"])), + NetIpv4IpLocalPortRange: to.Ptr(customSysctls["net.ipv4.ip_local_port_range"]), + NetIpv4TcpkeepaliveIntvl: to.Ptr(toolkit.StrToInt32(customSysctls["net.ipv4.tcp_keepalive_intvl"])), + }, + UlimitConfig: &aksnodeconfigv1.UlimitConfig{ + MaxLockedMemory: to.Ptr(customContainerdUlimits["LimitMEMLOCK"]), + NoFile: to.Ptr(customContainerdUlimits["LimitNOFILE"]), + }, + EnableSwapConfig: true, + SwapFileSize: swapFileSizeMB, + TransparentHugepageSupport: thpEnabled, + TransparentDefrag: thpDefrag, + } + config.KubeletConfig.EnableKubeletConfigFile = true + config.KubeletConfig.KubeletConfigFileConfig.FailSwapOn = to.Ptr(false) + }, + WaitForSSHAfterReboot: 10 * time.Minute, + Validator: func(ctx context.Context, s *Scenario) { + ValidateCustomLinuxOSConfigPersistsAfterReboot(ctx, s, customSysctls, customContainerdUlimits, swapFileSizeMB, thpEnabled, thpDefrag) + }, + }, + }) +} + func Test_Ubuntu2204_KubeletCustomConfig(t *testing.T) { RunScenario(t, &Scenario{ Tags: Tags{ diff --git a/e2e/test_helpers.go b/e2e/test_helpers.go index d2df3773e63..edf75233bf4 100644 --- a/e2e/test_helpers.go +++ b/e2e/test_helpers.go @@ -85,7 +85,7 @@ func RunScenario(t *testing.T, s *Scenario) { }) return } - if scriptlessUnsupported(s) { + if config.Config.DisableScriptless || scriptlessUnsupported(s) { require.NoError(t, runScenario(t, s)) return } @@ -292,7 +292,8 @@ func prepareAKSNode(ctx context.Context, s *Scenario) (*ScenarioVM, error) { s.BootstrapConfigMutator(s.Runtime.Cluster, nbc) } if s.AKSNodeConfigMutator != nil { - nodeconfig := nbcToAKSNodeConfigV1(nbc) + nodeconfig, err := nbcToAKSNodeConfigV1(nbc) + require.NoError(s.T, err) s.AKSNodeConfigMutator(s.Runtime.Cluster, nodeconfig) s.Runtime.AKSNodeConfig = nodeconfig @@ -304,7 +305,7 @@ func prepareAKSNode(ctx context.Context, s *Scenario) (*ScenarioVM, error) { // for scriptless phase 2.5, we are using nbc cse cmd for provisioning but passing aksnodeconfig and nbc cse cmd to compare env variables // scriptless tag means provisioning with aksnodeconfig is used - if !s.Tags.Scriptless && s.BootstrapConfigMutator != nil { + if !config.Config.DisableScriptless && !s.Tags.Scriptless && s.BootstrapConfigMutator != nil { nbc.EnableScriptlessNBCCSECmd = true } } diff --git a/e2e/validators.go b/e2e/validators.go index 4b8588a2481..cc5d5c2d629 100644 --- a/e2e/validators.go +++ b/e2e/validators.go @@ -250,7 +250,7 @@ func ValidateDirectoryContent(ctx context.Context, s *Scenario, path string, fil func ValidateSysctlConfig(ctx context.Context, s *Scenario, customSysctls map[string]string) { s.T.Helper() - keysToCheck := make([]string, len(customSysctls)) + keysToCheck := make([]string, 0, len(customSysctls)) for k := range customSysctls { keysToCheck = append(keysToCheck, k) } @@ -258,12 +258,106 @@ func ValidateSysctlConfig(ctx context.Context, s *Scenario, customSysctls map[st "set -ex", fmt.Sprintf("sudo sysctl %s | sed -E 's/([0-9])\\s+([0-9])/\\1 \\2/g'", strings.Join(keysToCheck, " ")), } - execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "systmctl command failed") + execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "sysctl command failed") for name, value := range customSysctls { require.Contains(s.T, execResult.stdout, fmt.Sprintf("%s = %v", name, value), "expected to find %s set to %v, but was not.\nStdout:\n%s", name, value, execResult.stdout) } } +func ValidateCustomLinuxOSConfigPersistsAfterReboot(ctx context.Context, s *Scenario, customSysctls map[string]string, customContainerdUlimits map[string]string, swapFileSizeMB int32, thpEnabled, thpDefrag string) { + s.T.Helper() + validateCustomLinuxOSConfig(ctx, s, customSysctls, customContainerdUlimits, swapFileSizeMB, thpEnabled, thpDefrag) + RebootVMAndWaitForSSH(ctx, s) + validateCustomLinuxOSConfig(ctx, s, customSysctls, customContainerdUlimits, swapFileSizeMB, thpEnabled, thpDefrag) +} + +func validateCustomLinuxOSConfig(ctx context.Context, s *Scenario, customSysctls map[string]string, customContainerdUlimits map[string]string, swapFileSizeMB int32, thpEnabled, thpDefrag string) { + s.T.Helper() + ValidateSysctlConfig(ctx, s, customSysctls) + ValidateUlimitSettings(ctx, s, customContainerdUlimits) + ValidateSwapFileConfig(ctx, s, swapFileSizeMB) + ValidateTransparentHugePageConfig(ctx, s, thpEnabled, thpDefrag) +} + +func ValidateTransparentHugePageConfig(ctx context.Context, s *Scenario, thpEnabled, thpDefrag string) { + s.T.Helper() + command := []string{"set -ex"} + if thpEnabled != "" { + command = append(command, + "cat /sys/kernel/mm/transparent_hugepage/enabled", + fmt.Sprintf("grep -Fq '[%s]' /sys/kernel/mm/transparent_hugepage/enabled", thpEnabled), + ) + } + if thpDefrag != "" { + command = append(command, + "cat /sys/kernel/mm/transparent_hugepage/defrag", + fmt.Sprintf("grep -Fq '[%s]' /sys/kernel/mm/transparent_hugepage/defrag", thpDefrag), + ) + } + + execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "transparent huge page configuration did not match expected values") +} + +func ValidateSwapFileConfig(ctx context.Context, s *Scenario, swapFileSizeMB int32) { + s.T.Helper() + if swapFileSizeMB <= 0 { + return + } + + command := []string{ + "set -ex", + "swapon --show --bytes", + "swap_file=$(awk '$3 == \"swap\" {print $1; exit}' /etc/fstab)", + "test -n \"${swap_file}\"", + "swapon --show --bytes | grep -F \"${swap_file}\"", + fmt.Sprintf("expected_bytes=$((%d * 1000 * 1000))", swapFileSizeMB), + "actual_bytes=$(stat -c %s \"${swap_file}\")", + "test \"${actual_bytes}\" -ge \"${expected_bytes}\"", + } + execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "swap file configuration did not match expected values") +} + +func RebootVMAndWaitForSSH(ctx context.Context, s *Scenario) { + s.T.Helper() + beforeRebootBootID := strings.TrimSpace(execScriptOnVMForScenarioValidateExitCode(ctx, s, "cat /proc/sys/kernel/random/boot_id", 0, "could not read boot ID before reboot").stdout) + execScriptOnVMForScenarioValidateExitCode(ctx, s, "sudo nohup sh -c 'sleep 1; systemctl reboot' >/dev/null 2>&1 &", 0, "failed to trigger VM reboot") + cleanupBastionTunnel(s.Runtime.VM.SSHClient) + s.Runtime.VM.SSHClient = nil + + waitTimeout := s.Config.WaitForSSHAfterReboot + if waitTimeout == 0 { + waitTimeout = 10 * time.Minute + } + + err := wait.PollUntilContextTimeout(ctx, 15*time.Second, waitTimeout, true, func(ctx context.Context) (bool, error) { + sshClient, err := DialSSHOverBastion(ctx, s.Runtime.Cluster.Bastion, s.Runtime.VM.PrivateIP, config.VMSSHPrivateKey) + if err != nil { + s.T.Logf("waiting for SSH after reboot: %v", err) + return false, nil + } + + bootIDCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + execResult, err := runSSHCommand(bootIDCtx, sshClient, "cat /proc/sys/kernel/random/boot_id", s.IsWindows()) + cancel() + if err != nil { + cleanupBastionTunnel(sshClient) + s.T.Logf("waiting for boot ID after reboot: %v", err) + return false, nil + } + + afterRebootBootID := strings.TrimSpace(execResult.stdout) + if afterRebootBootID == "" || afterRebootBootID == beforeRebootBootID { + cleanupBastionTunnel(sshClient) + s.T.Logf("waiting for VM reboot to complete: boot ID is still %q", afterRebootBootID) + return false, nil + } + + s.Runtime.VM.SSHClient = sshClient + return true, nil + }) + require.NoError(s.T, err, "timed out waiting for VM to reboot and accept SSH") +} + // ValidateNetworkInterfaceConfig validates network interface configuration settings using ethtool. // It identifies network interfaces with slot names matching the enP* pattern (same logic as the udev rule), // then verifies that each interface has the expected configuration settings (e.g., rx buffer size). diff --git a/e2e/vmss.go b/e2e/vmss.go index 5230cd14e0c..786bf7799ae 100644 --- a/e2e/vmss.go +++ b/e2e/vmss.go @@ -245,7 +245,7 @@ func createVMSSModel(ctx context.Context, s *Scenario) armcompute.VirtualMachine customData, err = injectWriteFilesEntriesToCustomData(customData, s.Config.CustomDataWriteFiles) require.NoError(s.T, err, "failed to inject customData write_files entries") } - if !scriptlessNBCCSECmdEnabled && s.VHD.SupportsScriptless() { + if !config.Config.DisableScriptless && !scriptlessNBCCSECmdEnabled && s.VHD.SupportsScriptless() { // Validate that the custom data doesn't contain any script content, // which indicates that the scriptless CSE is working as intended decodedCustomData, err := base64.StdEncoding.DecodeString(customData) diff --git a/parts/linux/cloud-init/artifacts/cse_config.sh b/parts/linux/cloud-init/artifacts/cse_config.sh index 359933b84ce..2c87414f18b 100755 --- a/parts/linux/cloud-init/artifacts/cse_config.sh +++ b/parts/linux/cloud-init/artifacts/cse_config.sh @@ -17,18 +17,98 @@ Environment="KUBE_API_SERVER_NAME=${API_SERVER_NAME}" EOF systemctlEnableAndStart reconcile-private-hosts 30 || exit $ERR_SYSTEMCTL_START_FAIL } + configureTransparentHugePage() { - ETC_SYSFS_CONF="/etc/sysfs.conf" + local etc_sysfs_conf="/etc/sysfs.conf" + + applyTransparentHugePageValues + if [ -n "${THP_ENABLED}" ]; then - echo "${THP_ENABLED}" > /sys/kernel/mm/transparent_hugepage/enabled - echo "kernel/mm/transparent_hugepage/enabled=${THP_ENABLED}" >> ${ETC_SYSFS_CONF} + printf 'kernel/mm/transparent_hugepage/enabled=%s\n' "${THP_ENABLED}" >> "${etc_sysfs_conf}" || exit "$ERR_SYSCTL_RELOAD" fi if [ -n "${THP_DEFRAG}" ]; then - echo "${THP_DEFRAG}" > /sys/kernel/mm/transparent_hugepage/defrag - echo "kernel/mm/transparent_hugepage/defrag=${THP_DEFRAG}" >> ${ETC_SYSFS_CONF} + printf 'kernel/mm/transparent_hugepage/defrag=%s\n' "${THP_DEFRAG}" >> "${etc_sysfs_conf}" || exit "$ERR_SYSCTL_RELOAD" + fi + reconcileTransparentHugePagePersistence +} + +applyTransparentHugePageValues() { + local thp_enabled_path="/sys/kernel/mm/transparent_hugepage/enabled" + local thp_defrag_path="/sys/kernel/mm/transparent_hugepage/defrag" + + if [ -n "${THP_ENABLED}" ]; then + printf '%s\n' "${THP_ENABLED}" > "${thp_enabled_path}" || exit "$ERR_SYSCTL_RELOAD" + fi + if [ -n "${THP_DEFRAG}" ]; then + printf '%s\n' "${THP_DEFRAG}" > "${thp_defrag_path}" || exit "$ERR_SYSCTL_RELOAD" + fi +} + +reconcileTransparentHugePagePersistence() { + if { [ -n "${THP_ENABLED}" ] || [ -n "${THP_DEFRAG}" ]; } && isMarinerOrAzureLinux "$OS" "$OS_VARIANT"; then + configureTransparentHugePageSystemdService fi } +configureTransparentHugePageSystemdService() { + local service_name="aks-transparent-hugepage" + local script_path="/opt/azure/containers/aks-transparent-hugepage.sh" + local config_dir="/opt/azure/containers/aks-transparent-hugepage" + local service_path="/etc/systemd/system/${service_name}.service" + + mkdir -p "$(dirname "${script_path}")" "${config_dir}" || exit "$ERR_SYSCTL_RELOAD" + if [ -n "${THP_ENABLED}" ]; then + printf '%s\n' "${THP_ENABLED}" | tee "${config_dir}/enabled" > /dev/null || exit "$ERR_SYSCTL_RELOAD" + else + rm -f "${config_dir}/enabled" || exit "$ERR_SYSCTL_RELOAD" + fi + if [ -n "${THP_DEFRAG}" ]; then + printf '%s\n' "${THP_DEFRAG}" | tee "${config_dir}/defrag" > /dev/null || exit "$ERR_SYSCTL_RELOAD" + else + rm -f "${config_dir}/defrag" || exit "$ERR_SYSCTL_RELOAD" + fi + + if ! tee "${script_path}" > /dev/null <<'EOF' +#!/bin/bash +set -e +config_dir="/opt/azure/containers/aks-transparent-hugepage" +thp_enabled_config="${config_dir}/enabled" +thp_defrag_config="${config_dir}/defrag" + +if [ -s "${thp_enabled_config}" ]; then + cat "${thp_enabled_config}" > /sys/kernel/mm/transparent_hugepage/enabled +fi +if [ -s "${thp_defrag_config}" ]; then + cat "${thp_defrag_config}" > /sys/kernel/mm/transparent_hugepage/defrag +fi +EOF + then + exit "$ERR_SYSCTL_RELOAD" + fi + chmod 0755 "${script_path}" || exit "$ERR_SYSCTL_RELOAD" + + if ! tee "${service_path}" > /dev/null </dev/null || stat -f "%Lp" "${file}" 2>/dev/null +} + +ensureSwapFileFstabEntry() { + local swap_location="$1" + local fstab_entry="${swap_location} none swap noauto,nofail 0 0" + local fstab_file="${2:-/etc/fstab}" + local fstab_dir + local fstab_mode + local temp_fstab + + fstab_dir="$(dirname "${fstab_file}")" + temp_fstab="$(mktemp "${fstab_dir}/fstab.XXXXXX")" || return 1 + fstab_mode="$(getFileMode "${fstab_file}")" || { + rm -f "${temp_fstab}" + return 1 + } + chmod "${fstab_mode}" "${temp_fstab}" || { + rm -f "${temp_fstab}" + return 1 + } + awk -v swap_location="${swap_location}" '$1 != swap_location { print }' "${fstab_file}" > "${temp_fstab}" || { + rm -f "${temp_fstab}" + return 1 + } + echo "${fstab_entry}" >> "${temp_fstab}" || { + rm -f "${temp_fstab}" + return 1 + } + mv "${temp_fstab}" "${fstab_file}" || { + rm -f "${temp_fstab}" + return 1 + } +} + +findExistingSwapFileLocation() { + local resource_disk_path + local swap_location + + if [ -L /dev/disk/azure/resource-part1 ]; then + resource_disk_path=$(findmnt -nr -o target -S "$(readlink -f /dev/disk/azure/resource-part1)" || true) + swap_location="${resource_disk_path}/swapfile" + if [ -n "${resource_disk_path}" ] && [ -f "${swap_location}" ]; then + echo "${swap_location}" + return 0 + fi + fi + + if [ -f /swapfile ]; then + echo "/swapfile" + return 0 + fi + + return 1 +} + +reconcileSwapFilePersistence() { + local swap_location="${1:-}" + + if [ -z "${swap_location}" ]; then + swap_location="$(findExistingSwapFileLocation || true)" + fi + + if [ -z "${swap_location}" ]; then + echo "No existing AKS swap file found; creating swap file for persistence reconciliation" + configureSwapFile + return 0 + fi + + ensureSwapFileFstabEntry "${swap_location}" || exit "$ERR_SWAP_CREATE_FAIL" + configureSwapFileSystemdService "${swap_location}" || exit "$ERR_SWAP_CREATE_FAIL" +} + configureSwapFile() { # https://learn.microsoft.com/en-us/troubleshoot/azure/virtual-machines/troubleshoot-device-names-problems#identify-disk-luns - swap_size_kb=$(expr ${SWAP_FILE_SIZE_MB} \* 1000) + swap_size_kb=$(expr "${SWAP_FILE_SIZE_MB}" \* 1000) swap_location="" # Attempt to use the resource disk if [ -L /dev/disk/azure/resource-part1 ]; then - resource_disk_path=$(findmnt -nr -o target -S $(readlink -f /dev/disk/azure/resource-part1)) - disk_free_kb=$(df ${resource_disk_path} | sed 1d | awk '{print $4}') - if [ "${disk_free_kb}" -gt "${swap_size_kb}" ]; then - echo "Will use resource disk for swap file" - swap_location=${resource_disk_path}/swapfile + resource_disk_path=$(findmnt -nr -o target -S "$(readlink -f /dev/disk/azure/resource-part1)" || true) + if [ -n "${resource_disk_path}" ]; then + disk_free_kb=$(df -P "${resource_disk_path}" | sed 1d | awk '{print $4}') + case "${disk_free_kb}" in + ''|*[!0-9]*) + echo "Could not determine free space on resource disk, attempting to fall back to OS disk..." + ;; + *) + if [ "${disk_free_kb}" -gt "${swap_size_kb}" ]; then + echo "Will use resource disk for swap file" + swap_location=${resource_disk_path}/swapfile + else + echo "Insufficient disk space on resource disk to create swap file: request ${swap_size_kb} free ${disk_free_kb}, attempting to fall back to OS disk..." + fi + ;; + esac else - echo "Insufficient disk space on resource disk to create swap file: request ${swap_size_kb} free ${disk_free_kb}, attempting to fall back to OS disk..." + echo "Could not determine resource disk mountpoint, attempting to fall back to OS disk..." fi fi @@ -81,12 +254,58 @@ configureSwapFile() { fi echo "Swap file will be saved to: ${swap_location}" - retrycmd_if_failure 24 5 25 fallocate -l ${swap_size_kb}K ${swap_location} || exit $ERR_SWAP_CREATE_FAIL - chmod 600 ${swap_location} - retrycmd_if_failure 24 5 25 mkswap ${swap_location} || exit $ERR_SWAP_CREATE_FAIL - retrycmd_if_failure 24 5 25 swapon ${swap_location} || exit $ERR_SWAP_CREATE_FAIL - retrycmd_if_failure 24 5 25 swapon --show | grep ${swap_location} || exit $ERR_SWAP_CREATE_FAIL - echo "${swap_location} none swap sw 0 0" >> /etc/fstab + retrycmd_if_failure 24 5 25 fallocate -l "${swap_size_kb}K" "${swap_location}" || exit "$ERR_SWAP_CREATE_FAIL" + chmod 600 "${swap_location}" + retrycmd_if_failure 24 5 25 mkswap "${swap_location}" || exit "$ERR_SWAP_CREATE_FAIL" + retrycmd_if_failure 24 5 25 swapon "${swap_location}" || exit "$ERR_SWAP_CREATE_FAIL" + swapFileIsActive "${swap_location}" || exit "$ERR_SWAP_CREATE_FAIL" + reconcileSwapFilePersistence "${swap_location}" || exit "$ERR_SWAP_CREATE_FAIL" +} + +configureSwapFileSystemdService() { + local swap_location="$1" + local service_name="aks-swapfile" + local script_path="/opt/azure/containers/aks-swapfile.sh" + local service_path="/etc/systemd/system/${service_name}.service" + local swap_mount_path + + swap_mount_path="$(dirname "${swap_location}")" + + mkdir -p "$(dirname "${script_path}")" || exit "$ERR_SWAP_CREATE_FAIL" + if ! tee "${script_path}" > /dev/null < /dev/null <&2 + return 0 + fi + cat > /dev/null + } + + chmod() { + echo "chmod $*" + [ "${FAIL_CHMOD:-}" = "true" ] && return 1 + return 0 + } + + systemctl() { + echo "systemctl $*" + [ "${1:-}" = "daemon-reload" ] && [ "${FAIL_DAEMON_RELOAD:-}" = "true" ] && return 1 + return 0 + } + + systemctlEnableAndStart() { + echo "systemctlEnableAndStart $*" + [ "${FAIL_SYSTEMCTL_ENABLE:-}" = "true" ] && return 1 + return 0 + } + + BeforeEach 'setup_thp_service' + AfterEach 'cleanup_thp_service' + + It 'writes helper files, reloads systemd, and enables the service' + When run configureTransparentHugePageSystemdService + + The status should be success + The output should include "mkdir -p /opt/azure/containers /opt/azure/containers/aks-transparent-hugepage" + The output should include "chmod 0755 /opt/azure/containers/aks-transparent-hugepage.sh" + The output should include "systemctl daemon-reload" + The output should include "systemctlEnableAndStart aks-transparent-hugepage 30" + End + + It 'exits when helper directory creation fails' + FAIL_MKDIR="true" + + When run configureTransparentHugePageSystemdService + + The status should equal "$ERR_SYSCTL_RELOAD" + The output should include "mkdir -p /opt/azure/containers" + The output should not include "chmod" + The output should not include "systemctl daemon-reload" + End + + It 'exits when helper script write fails' + FAIL_TEE_PATH="/opt/azure/containers/aks-transparent-hugepage.sh" + + When run configureTransparentHugePageSystemdService + + The status should equal "$ERR_SYSCTL_RELOAD" + The output should include "mkdir -p /opt/azure/containers" + The output should not include "chmod" + The output should not include "systemctl daemon-reload" + End + + It 'exits when helper script chmod fails' + FAIL_CHMOD="true" + + When run configureTransparentHugePageSystemdService + + The status should equal "$ERR_SYSCTL_RELOAD" + The output should include "chmod 0755 /opt/azure/containers/aks-transparent-hugepage.sh" + The output should not include "systemctl daemon-reload" + End + + It 'exits when service unit write fails' + FAIL_TEE_PATH="/etc/systemd/system/aks-transparent-hugepage.service" + + When run configureTransparentHugePageSystemdService + + The status should equal "$ERR_SYSCTL_RELOAD" + The output should include "chmod 0755 /opt/azure/containers/aks-transparent-hugepage.sh" + The output should not include "systemctl daemon-reload" + End + + It 'exits when systemd daemon reload fails' + FAIL_DAEMON_RELOAD="true" + + When run configureTransparentHugePageSystemdService + + The status should equal "$ERR_SYSTEMCTL_START_FAIL" + The output should include "systemctl daemon-reload" + The output should not include "systemctlEnableAndStart" + End + + It 'does not embed raw THP values in the generated helper script' + THP_ENABLED='never"; touch /tmp/aks-thp-injection #' + CAPTURE_TEE_PATH="/opt/azure/containers/aks-transparent-hugepage.sh" + + When run configureTransparentHugePageSystemdService + + The status should be success + The output should include "systemctlEnableAndStart aks-transparent-hugepage 30" + The error should include 'cat "${thp_enabled_config}" > /sys/kernel/mm/transparent_hugepage/enabled' + The error should not include "touch /tmp/aks-thp-injection" + End + End + + Describe 'swapFileIsActive' + swapon() { + if [ "$*" != "--show --noheadings" ]; then + return 1 + fi + printf '%b' "${SWAPON_OUTPUT}" + } + + It 'matches an active swap file when swapon output has leading whitespace' + SWAPON_OUTPUT=' /swapfile\n' + + When call swapFileIsActive "/swapfile" + The status should be success + End + + It 'matches only the exact swap file path' + SWAPON_OUTPUT=' /swapfile-extra\n' + + When call swapFileIsActive "/swapfile" + The status should be failure + End + End + + Describe 'ensureSwapFileFstabEntry' + setup() { + TEST_FSTAB_DIR="$(mktemp -d)" + TEST_FSTAB_FILE="${TEST_FSTAB_DIR}/fstab" + : > "${TEST_FSTAB_FILE}" + } + + cleanup() { + rm -rf "${TEST_FSTAB_DIR}" + unset TEST_FSTAB_FILE + unset TEST_FSTAB_DIR + unset FAIL_MV + } + + BeforeEach 'setup' + AfterEach 'cleanup' + + mv() { + if [ "${FAIL_MV:-false}" = "true" ]; then + return 1 + fi + + command mv "$@" + } + + It 'replaces existing fstab entries for the same swap file' + chmod 0644 "${TEST_FSTAB_FILE}" + printf '/swapfile none swap sw 0 0\n/other none swap sw 0 0\n/swapfile none swap defaults 0 0\n' > "${TEST_FSTAB_FILE}" + expected_fstab='/other none swap sw 0 0 +/swapfile none swap noauto,nofail 0 0' + + When call ensureSwapFileFstabEntry "/swapfile" "${TEST_FSTAB_FILE}" + + The status should be success + The contents of file "${TEST_FSTAB_FILE}" should equal "${expected_fstab}" + End + + It 'preserves the fstab file mode when replacing entries' + chmod 0640 "${TEST_FSTAB_FILE}" + printf '/other none swap sw 0 0\n' > "${TEST_FSTAB_FILE}" + + When call ensureSwapFileFstabEntry "/swapfile" "${TEST_FSTAB_FILE}" + + The status should be success + The path "${TEST_FSTAB_FILE}" should be file + The result of function check_test_fstab_permissions should equal "0640" + End + + It 'keeps one canonical fstab entry when it already exists' + printf '/other none swap sw 0 0\n/swapfile none swap noauto,nofail 0 0\n' > "${TEST_FSTAB_FILE}" + expected_fstab='/other none swap sw 0 0 +/swapfile none swap noauto,nofail 0 0' + + When call ensureSwapFileFstabEntry "/swapfile" "${TEST_FSTAB_FILE}" + + The status should be success + The contents of file "${TEST_FSTAB_FILE}" should equal "${expected_fstab}" + End + + It 'leaves the existing fstab untouched when atomic replace fails' + printf '/other none swap sw 0 0\n' > "${TEST_FSTAB_FILE}" + FAIL_MV=true + + When call ensureSwapFileFstabEntry "/swapfile" "${TEST_FSTAB_FILE}" + + The status should be failure + The contents of file "${TEST_FSTAB_FILE}" should equal '/other none swap sw 0 0' + End + End + Describe 'logGPUDriverPrebakeReadiness' It 'reports marker_present=false when no prebake marker exists' GPU_DKMS_MARKER_FILE="$(mktemp)"; rm -f "${GPU_DKMS_MARKER_FILE}" @@ -2683,4 +2904,144 @@ EOF The output should not include 'nvidia-device-plugin' End End + + Describe 'configureSwapFile' + SWAP_FILE_SIZE_MB=1 + + function [ { + if test "$1" = "-L" && test "$2" = "/dev/disk/azure/resource-part1"; then + return 0 + fi + local last_arg="" + for last_arg in "$@"; do :; done + if test "${last_arg}" = "]"; then + command [ "$@" + else + command [ "$@" ] + fi + } + + readlink() { + case "$2" in + /dev/disk/azure/resource-part1) echo "/dev/sdb1" ;; + /dev/disk/azure/root) echo "/dev/sda1" ;; + esac + } + + retrycmd_if_failure() { + echo "retrycmd_if_failure $*" + } + + chmod() { + echo "chmod $*" + } + + swapFileIsActive() { + echo "swapFileIsActive $1" + } + + reconcileSwapFilePersistence() { + echo "reconcileSwapFilePersistence $1" + } + + It 'falls back to OS disk when resource disk mountpoint cannot be determined' + findmnt() { + return 1 + } + df() { + printf 'Filesystem 1024-blocks Used Available Capacity Mounted on\n/dev/sda1 1000000 0 1000000 0%% /\n' + } + + When call configureSwapFile + + The status should be success + The output should include "Could not determine resource disk mountpoint, attempting to fall back to OS disk..." + The output should include "Will use OS disk for swap file" + The output should include "Swap file will be saved to: /swapfile" + The output should include "reconcileSwapFilePersistence /swapfile" + End + + It 'falls back to OS disk when resource disk free space cannot be determined' + findmnt() { + echo "/mnt/resource" + } + df() { + case "$2" in + /mnt/resource) + printf 'Filesystem 1024-blocks Used Available Capacity Mounted on\n' + ;; + /) + printf 'Filesystem 1024-blocks Used Available Capacity Mounted on\n/dev/sda1 1000000 0 1000000 0%% /\n' + ;; + esac + } + + When call configureSwapFile + + The status should be success + The output should include "Could not determine free space on resource disk, attempting to fall back to OS disk..." + The output should include "Will use OS disk for swap file" + The output should include "Swap file will be saved to: /swapfile" + The output should include "reconcileSwapFilePersistence /swapfile" + End + End + + Describe 'reconcileSwapFilePersistence' + findExistingSwapFileLocation() { + return 1 + } + + configureSwapFile() { + echo "createSwapFile" + } + + ensureSwapFileFstabEntry() { + echo "ensureSwapFileFstabEntry $1" + } + + configureSwapFileSystemdService() { + echo "configureSwapFileSystemdService $1" + } + + It 'creates the requested swap file when no existing swap file is present' + When call reconcileSwapFilePersistence + + The status should be success + The output should include "No existing AKS swap file found; creating swap file for persistence reconciliation" + The output should include "createSwapFile" + The output should not include "ensureSwapFileFstabEntry" + The output should not include "configureSwapFileSystemdService" + End + + It 'reconciles persistence for an existing swap file without recreating it' + When call reconcileSwapFilePersistence "/swapfile" + + The status should be success + The output should include "ensureSwapFileFstabEntry /swapfile" + The output should include "configureSwapFileSystemdService /swapfile" + The output should not include "createSwapFile" + End + + It 'exits when fstab reconciliation fails' + ensureSwapFileFstabEntry() { + return 1 + } + + When run reconcileSwapFilePersistence "/swapfile" + + The status should equal "$ERR_SWAP_CREATE_FAIL" + The output should not include "configureSwapFileSystemdService" + End + + It 'exits when swap systemd service reconciliation fails' + configureSwapFileSystemdService() { + return 1 + } + + When run reconcileSwapFilePersistence "/swapfile" + + The status should equal "$ERR_SWAP_CREATE_FAIL" + The output should include "ensureSwapFileFstabEntry /swapfile" + End + End End diff --git a/spec/parts/linux/cloud-init/artifacts/cse_main_disable_modules_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_main_disable_modules_spec.sh index 9329541661e..6eddd14665a 100644 --- a/spec/parts/linux/cloud-init/artifacts/cse_main_disable_modules_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_main_disable_modules_spec.sh @@ -461,4 +461,15 @@ Describe 'CVE kernel module mitigation phase coverage' When call phase_body "nodePrep" The output should include "reconcileVulnerableKernelModuleMitigation" End + + It 'configures transparent huge page from basePrep so VHD bakes keep the intended state' + When call phase_body "basePrep" + The output should include "configureTransparentHugePage" + End + + It 'applies and reconciles transparent huge page from nodePrep for PIS and already-released VHDs' + When call phase_body "nodePrep" + The output should include "applyTransparentHugePageValues" + The output should include "reconcileTransparentHugePagePersistence" + End End From 4d58a06091d8875157dc56764dbbbcd3a421d8f6 Mon Sep 17 00:00:00 2001 From: Tim Wright Date: Fri, 7 Aug 2026 09:02:48 +1200 Subject: [PATCH 10/38] fix: update pause to 3.10.2 (#9128) --- e2e/kubelet/README.md | 2 +- e2e/node_config.go | 8 ++++---- image-fetcher/main.go | 2 +- parts/common/components.json | 6 ++++-- .../artifacts/ubuntu/gb/containerd-nvidia.toml | 2 +- pkg/agent/datamodel/mocks.go | 2 +- pkg/agent/datamodel/types.go | 2 +- .../linux/cloud-init/artifacts/cse_config_spec.sh | 12 ++++++------ staging/cse/windows/containerdfunc.tests.ps1 | 4 ++-- staging/cse/windows/kubernetesfunc.ps1 | 2 +- .../cse/windows/networkisolatedclusterfunc.tests.ps1 | 10 +++++----- 11 files changed, 27 insertions(+), 25 deletions(-) diff --git a/e2e/kubelet/README.md b/e2e/kubelet/README.md index e7a48355ea3..01147aff586 100644 --- a/e2e/kubelet/README.md +++ b/e2e/kubelet/README.md @@ -131,7 +131,7 @@ map[string]string{ "--one-output": "\"false\"", "--oom-score-adj": "\"-999\"", "--pod-cidr": "\"\"", - "--pod-infra-container-image": "\"registry.k8s.io/pause:3.8\"", + "--pod-infra-container-image": "\"registry.k8s.io/pause:3.10.2\"", "--pod-manifest-path": "\"\"", "--pod-max-pids": "\"-1\"", "--pods-per-core": "\"0\"", diff --git a/e2e/node_config.go b/e2e/node_config.go index 1bd82d24244..52cbef44d28 100644 --- a/e2e/node_config.go +++ b/e2e/node_config.go @@ -664,7 +664,7 @@ func baseTemplateLinux(t testing.TB, location string, k8sVersion string, arch st ContainerdDownloadURLBase: "https://storage.googleapis.com/cri-containerd-release/", CSIProxyDownloadURL: "https://packages.aks.azure.com/csi-proxy/v0.1.0/binaries/csi-proxy.tar.gz", WindowsProvisioningScriptsPackageURL: "https://packages.aks.azure.com/aks-engine/windows/provisioning/signedscripts-v0.2.2.zip", - WindowsPauseImageURL: "mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.1", + WindowsPauseImageURL: "mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.2", AlwaysPullWindowsPauseImage: false, CseScriptsPackageURL: "https://packages.aks.azure.com/aks/windows/cse/", CNIARM64PluginsDownloadURL: "https://packages.aks.azure.com/cni-plugins/v0.8.7/binaries/cni-plugins-linux-arm64-v0.8.7.tgz", @@ -676,7 +676,7 @@ func baseTemplateLinux(t testing.TB, location string, k8sVersion string, arch st OSImageConfig: map[datamodel.Distro]datamodel.AzureOSImageConfig(nil), }, K8sComponents: &datamodel.K8sComponents{ - PodInfraContainerImageURL: "mcr.microsoft.com/oss/v2/kubernetes/pause:3.6", + PodInfraContainerImageURL: "mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.2", HyperkubeImageURL: "mcr.microsoft.com/oss/kubernetes/", WindowsPackageURL: "windowspackage", LinuxCredentialProviderURL: "", @@ -938,7 +938,7 @@ func baseTemplateWindows(t testing.TB, location string) *datamodel.NodeBootstrap WindowsDockerVersion: "", WindowsImageSourceURL: "", WindowsOffer: "aks-windows", - WindowsPauseImageURL: "mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.1", + WindowsPauseImageURL: "mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.2", WindowsPublisher: "microsoft-aks", WindowsSku: "", }, @@ -981,7 +981,7 @@ DXRqvV7TWO2hndliQq3BW385ZkiephlrmpUVM= r2k1@arturs-mbp.lan`, // VnetCNIARM64LinuxPluginsDownloadURL: "https://packages.aks.azure.com/azure-cni/v1.4.13/binaries/azure-vnet-cni-linux-arm64-v1.4.14.tgz", // VnetCNILinuxPluginsDownloadURL: "https://packages.aks.azure.com/azure-cni/v1.1.3/binaries/azure-vnet-cni-linux-amd64-v1.1.3.tgz", VnetCNIWindowsPluginsDownloadURL: "https://packages.aks.azure.com/azure-cni/v1.6.21/binaries/azure-vnet-cni-windows-amd64-v1.6.21.zip", - WindowsPauseImageURL: "mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.1", + WindowsPauseImageURL: "mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.2", WindowsProvisioningScriptsPackageURL: "", WindowsTelemetryGUID: "fb801154-36b9-41bc-89c2-f4d4f05472b0", }, diff --git a/image-fetcher/main.go b/image-fetcher/main.go index 40005911584..90af856e320 100644 --- a/image-fetcher/main.go +++ b/image-fetcher/main.go @@ -24,7 +24,7 @@ const ( func main() { if len(os.Args) < 2 { fmt.Fprintf(os.Stderr, "Usage: %s [image-ref...]\n", os.Args[0]) - fmt.Fprintf(os.Stderr, "Example: %s mcr.microsoft.com/oss/kubernetes/pause:3.9\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "Example: %s mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.2\n", os.Args[0]) os.Exit(1) } diff --git a/parts/common/components.json b/parts/common/components.json index b99b0eb289b..a7bdb04b5ea 100644 --- a/parts/common/components.json +++ b/parts/common/components.json @@ -102,13 +102,15 @@ "multiArchVersionsV2": [ { "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes/pause", - "latestVersion": "3.6" + "latestVersion": "3.10.2", + "previousLatestVersion": "3.6" } ], "windowsVersions": [ { "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes/pause", - "latestVersion": "3.10.1" + "latestVersion": "3.10.2", + "previousLatestVersion": "3.10.1" } ] }, diff --git a/parts/linux/cloud-init/artifacts/ubuntu/gb/containerd-nvidia.toml b/parts/linux/cloud-init/artifacts/ubuntu/gb/containerd-nvidia.toml index 88aa0fa0222..4f3771ab9a8 100644 --- a/parts/linux/cloud-init/artifacts/ubuntu/gb/containerd-nvidia.toml +++ b/parts/linux/cloud-init/artifacts/ubuntu/gb/containerd-nvidia.toml @@ -7,7 +7,7 @@ version = 2 [plugins] [plugins."io.containerd.grpc.v1.cri"] - sandbox_image = "mcr.microsoft.com/oss/kubernetes/pause:3.6" + sandbox_image = "mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.2" [plugins."io.containerd.grpc.v1.cri".containerd] default_runtime_name = "nvidia" diff --git a/pkg/agent/datamodel/mocks.go b/pkg/agent/datamodel/mocks.go index 96f27ae4184..eacb54c8673 100644 --- a/pkg/agent/datamodel/mocks.go +++ b/pkg/agent/datamodel/mocks.go @@ -129,7 +129,7 @@ var ( ContainerdDownloadURLBase: "https://storage.googleapis.com/cri-containerd-release/", CSIProxyDownloadURL: "https://acs-mirror.azureedge.net/csi-proxy/v0.1.0/binaries/csi-proxy.tar.gz", WindowsProvisioningScriptsPackageURL: "https://acs-mirror.azureedge.net/aks-engine/windows/provisioning/signedscripts-v0.2.2.zip", - WindowsPauseImageURL: "mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.1", + WindowsPauseImageURL: "mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.2", AlwaysPullWindowsPauseImage: false, CseScriptsPackageURL: "https://acs-mirror.azureedge.net/aks/windows/cse/csescripts-v0.0.1.zip", CNIARM64PluginsDownloadURL: "https://acs-mirror.azureedge.net/cni-plugins/v0.8.7/binaries/cni-plugins-linux-arm64-v0.8.7.tgz", diff --git a/pkg/agent/datamodel/types.go b/pkg/agent/datamodel/types.go index 69f013e864b..a0d405bdb33 100644 --- a/pkg/agent/datamodel/types.go +++ b/pkg/agent/datamodel/types.go @@ -1696,7 +1696,7 @@ func FormatProdFQDNByLocation(fqdnPrefix string, location string, cloudSpecConfi type K8sComponents struct { // Full path to the "pause" image. Used for --pod-infra-container-image. - // For example: "mcr.microsoft.com/oss/v2/kubernetes/pause:3.6". + // For example: "mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.2". PodInfraContainerImageURL string // Full path to the hyperkube image. diff --git a/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh index af9784f5da8..c5fc8c01f49 100755 --- a/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh @@ -2758,36 +2758,36 @@ OVERRIDE_EOF ERR_PULL_POD_INFRA_CONTAINER_IMAGE=1 It 'should use MCR_REPOSITORY_BASE for image replacement when set' - get_sandbox_image() { echo "mcr.microsoft.us/oss/v2/kubernetes/pause:3.10.1"; } + get_sandbox_image() { echo "mcr.microsoft.us/oss/v2/kubernetes/pause:3.10.2"; } MCR_REPOSITORY_BASE="mcr.microsoft.us" BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER="myacr.azurecr.io/aks-managed-repository" When call ensurePodInfraContainerImage The status should be success - The output should include "Pulling with authentication for myacr.azurecr.io/aks-managed-repository/oss/v2/kubernetes/pause:3.10.1" + The output should include "Pulling with authentication for myacr.azurecr.io/aks-managed-repository/oss/v2/kubernetes/pause:3.10.2" End It 'should fall back to mcr.microsoft.com when MCR_REPOSITORY_BASE is unset' - get_sandbox_image() { echo "mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.1"; } + get_sandbox_image() { echo "mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.2"; } MCR_REPOSITORY_BASE="" BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER="myacr.azurecr.io/aks-managed-repository" When call ensurePodInfraContainerImage The status should be success - The output should include "Pulling with authentication for myacr.azurecr.io/aks-managed-repository/oss/v2/kubernetes/pause:3.10.1" + The output should include "Pulling with authentication for myacr.azurecr.io/aks-managed-repository/oss/v2/kubernetes/pause:3.10.2" End It 'should handle MCR_REPOSITORY_BASE with trailing slash' - get_sandbox_image() { echo "mcr.microsoft.us/oss/v2/kubernetes/pause:3.10.1"; } + get_sandbox_image() { echo "mcr.microsoft.us/oss/v2/kubernetes/pause:3.10.2"; } MCR_REPOSITORY_BASE="mcr.microsoft.us/" BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER="myacr.azurecr.io/aks-managed-repository" When call ensurePodInfraContainerImage The status should be success - The output should include "Pulling with authentication for myacr.azurecr.io/aks-managed-repository/oss/v2/kubernetes/pause:3.10.1" + The output should include "Pulling with authentication for myacr.azurecr.io/aks-managed-repository/oss/v2/kubernetes/pause:3.10.2" End End diff --git a/staging/cse/windows/containerdfunc.tests.ps1 b/staging/cse/windows/containerdfunc.tests.ps1 index ff88cfc9997..c510d4f6df0 100644 --- a/staging/cse/windows/containerdfunc.tests.ps1 +++ b/staging/cse/windows/containerdfunc.tests.ps1 @@ -85,7 +85,7 @@ Describe "Containerd Functions Tests" { $containerdDir = "$PSScriptRoot\containerdfunc.tests.suites" $cniBinDir = 'C:/cni/bin' $cniConfDir = 'C:/cni/conf' - $pauseImage = 'mcr.microsoft.com/oss/v2/kubernetes/pause:3.6' + $pauseImage = 'mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.2' $global:KubeClusterConfigPath = [Io.path]::Combine("", "kubeclusterconfig.json") $global:ContainerdInstallLocation = $containerdDir @@ -146,7 +146,7 @@ Describe "Containerd Functions Tests" { $containerdDir = "$PSScriptRoot\containerdfunc.tests.suites" $cniBinDir = 'C:/cni/bin' $cniConfDir = 'C:/cni/conf' - $pauseImage = 'mcr.microsoft.com/oss/v2/kubernetes/pause:3.6' + $pauseImage = 'mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.2' $global:KubeClusterConfigPath = [Io.path]::Combine("", "kubeclusterconfig.json") $global:ContainerdInstallLocation = $containerdDir diff --git a/staging/cse/windows/kubernetesfunc.ps1 b/staging/cse/windows/kubernetesfunc.ps1 index 9a613380d30..50d1126cb8a 100644 --- a/staging/cse/windows/kubernetesfunc.ps1 +++ b/staging/cse/windows/kubernetesfunc.ps1 @@ -115,7 +115,7 @@ function Write-KubeClusterConfig { $Global:ClusterConfiguration | Add-Member -MemberType NoteProperty -Name Cri -Value @{ Name = "containerd"; Images = @{ - # e.g. "mcr.microsoft.com/oss/v2/kubernetes/pause:3.6" + # e.g. "mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.2" "Pause" = $global:WindowsPauseImageURL } } diff --git a/staging/cse/windows/networkisolatedclusterfunc.tests.ps1 b/staging/cse/windows/networkisolatedclusterfunc.tests.ps1 index 6c3129ab67a..2c7613bae6d 100644 --- a/staging/cse/windows/networkisolatedclusterfunc.tests.ps1 +++ b/staging/cse/windows/networkisolatedclusterfunc.tests.ps1 @@ -143,7 +143,7 @@ Describe "Set-PodInfraContainerImage" { { "Cri": { "Images": { - "Pause": "mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.1" + "Pause": "mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.2" } } } @@ -176,7 +176,7 @@ Describe "Set-PodInfraContainerImage" { It "returns early when image already exists locally" { $script:CtrExeMock = { param($Args) - return @("mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.1") + return @("mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.2") } function global:Mock-OrasCli { @@ -241,7 +241,7 @@ Describe "Set-PodInfraContainerImage" { $global:MCRRepositoryBase = $null { Set-PodInfraContainerImage } | Should -Not -Throw - $script:orasImageArg | Should -Be "myacr.azurecr.io/aks-managed-repository/oss/v2/kubernetes/pause:3.10.1" + $script:orasImageArg | Should -Be "myacr.azurecr.io/aks-managed-repository/oss/v2/kubernetes/pause:3.10.2" } It "should use MCRRepositoryBase (and trim trailing slash) for image replacement" { @@ -250,7 +250,7 @@ Describe "Set-PodInfraContainerImage" { { "Cri": { "Images": { - "Pause": "mcr.microsoft.us/oss/v2/kubernetes/pause:3.10.1" + "Pause": "mcr.microsoft.us/oss/v2/kubernetes/pause:3.10.2" } } } @@ -268,7 +268,7 @@ Describe "Set-PodInfraContainerImage" { $global:MCRRepositoryBase = "mcr.microsoft.us/" { Set-PodInfraContainerImage } | Should -Not -Throw - $script:orasImageArg | Should -Be "myacr.azurecr.io/aks-managed-repository/oss/v2/kubernetes/pause:3.10.1" + $script:orasImageArg | Should -Be "myacr.azurecr.io/aks-managed-repository/oss/v2/kubernetes/pause:3.10.2" } } From 3881174c4a1c81a9e9628fcd030deb0dcdfd83f8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 21:05:17 -0700 Subject: [PATCH 11/38] chore(deps): bump github/codeql-action from 4.37.3 to 4.37.6 (#9154) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql-analysis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index b401379da51..cffb0d14d5f 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -48,7 +48,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v4.37.3 + uses: github/codeql-action/init@v4.37.6 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -62,7 +62,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v4.37.3 + uses: github/codeql-action/autobuild@v4.37.6 # ℹ️ Command-line programs to run using the OS shell. # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun @@ -75,4 +75,4 @@ jobs: # ./location_of_script_within_repo/buildscript.sh - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v4.37.3 + uses: github/codeql-action/analyze@v4.37.6 From bbbdb538ff63daed6fa716bcba3da2c7934779a1 Mon Sep 17 00:00:00 2001 From: Tim Wright Date: Fri, 7 Aug 2026 16:08:49 +1200 Subject: [PATCH 12/38] fix: rename template file (#9155) --- .github/copilot-instructions.md | 6 ++-- AGENTS.md | 30 ++++++++++++++----- ...s1 => kuberneteswindowssetup.ps1.template} | 0 parts/windows/windowscsehelper.ps1 | 2 +- pkg/agent/const.go | 2 +- staging/cse/windows/README | 2 +- 6 files changed, 29 insertions(+), 13 deletions(-) rename parts/windows/{kuberneteswindowssetup.ps1 => kuberneteswindowssetup.ps1.template} (100%) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index c056cf6e093..34e8e69549d 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -17,9 +17,9 @@ Windows VHD is configured through [VHD](./vhdbuilder/packer/windows/windows-vhd- [apiserver](./apiserver/) is `go` based webserver. It receives request from external client and generates CSE and CustomData to be used on the VHD when a new node is created / provisioned. -windows generates its CSE package using [script](./parts/windows/kuberneteswindowssetup.ps1). +windows generates its CSE package using [script](./parts/windows/kuberneteswindowssetup.ps1.template). -Both CSE scripts run in two phases — Windows `BasePrep`/`NodePrep` (`kuberneteswindowssetup.ps1`) and Linux `basePrep`/`nodePrep` (`cse_main.sh`). `basePrep` is gated only by the `base_prep.complete` marker, so it is **skipped on a PIS-cached VHD** (the marker is baked in); `nodePrep` runs whenever `PreProvisionOnly` is false — every real node. So node-specific, secret, or expiring data (e.g. the TLS bootstrap token / kubeconfig) must be written in `nodePrep`, not `basePrep`, or the baked copy goes stale. On PIS the phases are separate VM runs — variables re-initialize from the real node's live CustomData, so `nodePrep` can't rely on `basePrep` state. (Non-PIS: both run sequentially in one execution, no reboot between phases.) +Both CSE scripts run in two phases — Windows `BasePrep`/`NodePrep` (`kuberneteswindowssetup.ps1.template`) and Linux `basePrep`/`nodePrep` (`cse_main.sh`). `basePrep` is gated only by the `base_prep.complete` marker, so it is **skipped on a PIS-cached VHD** (the marker is baked in); `nodePrep` runs whenever `PreProvisionOnly` is false — every real node. So node-specific, secret, or expiring data (e.g. the TLS bootstrap token / kubeconfig) must be written in `nodePrep`, not `basePrep`, or the baked copy goes stale. On PIS the phases are separate VM runs — variables re-initialize from the real node's live CustomData, so `nodePrep` can't rely on `basePrep` state. (Non-PIS: both run sequentially in one execution, no reboot between phases.) The webserver is also used to determine the latest version of Linux VHDs available for provisioning within AKS clusters. @@ -145,7 +145,7 @@ Analyze PRs for these compatibility scenarios: - Hardcoded paths that differ between deployment modes **4. PIS / VHD Caching — basePrep vs nodePrep split (Windows + Linux)** -- **Context**: PIS bakes a VHD from a temporary VM, then boots many real nodes from it. Same model in Windows `parts/windows/kuberneteswindowssetup.ps1` (`BasePrep`/`NodePrep`) and Linux `parts/linux/cloud-init/artifacts/cse_main.sh` (`basePrep`/`nodePrep`): +- **Context**: PIS bakes a VHD from a temporary VM, then boots many real nodes from it. Same model in Windows `parts/windows/kuberneteswindowssetup.ps1.template` (`BasePrep`/`NodePrep`) and Linux `parts/linux/cloud-init/artifacts/cse_main.sh` (`basePrep`/`nodePrep`): - `basePrep` — gated only by the `base_prep.complete` marker (`C:\AzureData\` Windows, `/opt/azure/containers/` Linux). **Skipped on PIS real nodes** because the marker is baked into the VHD. - `nodePrep` — runs whenever `PreProvisionOnly` is false (every real node); skipped only on the bake VM. - The bake run sets `PreProvisionOnly=true` (`{{GetPreProvisionOnly}}`) and writes the marker after `basePrep` succeeds (Windows `finally`; Linux `cse_start.sh`). diff --git a/AGENTS.md b/AGENTS.md index 93108d87d1f..13b0e949e10 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -17,9 +17,9 @@ Windows VHD is configured through [VHD](./vhdbuilder/packer/windows/windows-vhd- [apiserver](./apiserver/) is `go` based webserver. It receives request from external client and generates CSE and CustomData to be used on the VHD when a new node is created / provisioned. -windows generates its CSE package using [script](./parts/windows/kuberneteswindowssetup.ps1). +windows generates its CSE package using [script](./parts/windows/kuberneteswindowssetup.ps1.template). -Both CSE scripts run in two phases — Windows `BasePrep`/`NodePrep` (`kuberneteswindowssetup.ps1`) and Linux `basePrep`/`nodePrep` (`cse_main.sh`). `basePrep` is gated only by the `base_prep.complete` marker, so it is **skipped on a PIS-cached VHD** (the marker is baked in); `nodePrep` runs whenever `PreProvisionOnly` is false — every real node. So node-specific, secret, or expiring data (e.g. the TLS bootstrap token / kubeconfig) must be written in `nodePrep`, not `basePrep`, or the baked copy goes stale. On PIS the phases are separate VM runs — variables re-initialize from the real node's live CustomData, so `nodePrep` can't rely on `basePrep` state. (Non-PIS: both run sequentially in one execution, no reboot between phases.) +Both CSE scripts run in two phases — Windows `BasePrep`/`NodePrep` (`kuberneteswindowssetup.ps1.template`) and Linux `basePrep`/`nodePrep` (`cse_main.sh`). `basePrep` is gated only by the `base_prep.complete` marker, so it is **skipped on a PIS-cached VHD** (the marker is baked in); `nodePrep` runs whenever `PreProvisionOnly` is false — every real node. So node-specific, secret, or expiring data (e.g. the TLS bootstrap token / kubeconfig) must be written in `nodePrep`, not `basePrep`, or the baked copy goes stale. On PIS the phases are separate VM runs — variables re-initialize from the real node's live CustomData, so `nodePrep` can't rely on `basePrep` state. (Non-PIS: both run sequentially in one execution, no reboot between phases.) The webserver is also used to determine the latest version of Linux VHDs available for provisioning within AKS clusters. @@ -27,8 +27,6 @@ The webserver is also used to determine the latest version of Linux VHDs availab [parts](./parts/) serves both AgentBaker Service and VHD build. AgentBaker service and VHDs are coupled because of this shared component. When building VHD, packer maps and renames scripts from [parts](./parts/) depending on the OS / versions. The mappings can be found at [packer](./vhdbuilder/packer/). -> **IMPORTANT**: When making changes to files in the `parts` or `pkg` directories, you must run `make generate` afterward to regenerate the snapshot test data. This ensures consistency between the code and tests and prevents regressions. - Windows uses a different folder [cse](./staging/cse/windows/) for almost the same purpose. There are subtle differences as windows CSEs can be downloaded as a zip file during provisioning time due to restrictions on the file size on Windows system, while for linux based systems the cse/custom data are dropped in during provisioning time. ## Deployment and Release @@ -61,7 +59,7 @@ The operational goals of this project are: When making changes, reason whether the file is used in VHD building stage, or provision stage, or both. Make sure the changes are valid in its life stage. as an example, [windows-vhd-configuration.ps1](./vhdbuilder/packer/windows/windows-vhd-configuration.ps1) defines container images to be cached in VHD, while [configure-windows-vhd.ps1](./vhdbuilder/packer/windows/configure-windows-vhd.ps1) executes commands at provision time. -One way to debug / explore / just for fun is to run [e2e](./e2e/) tests. To run locally, follow the readme file under that folder. +One way to debug / explore / just for fun is to run [e2e](./e2e/) tests. To run locally, follow the readme file under that folder. The SRE guidelines ground other coding guidelines and practices. @@ -94,6 +92,7 @@ When reviewing pull requests, perform breaking change analysis to prevent regres Analyze PRs for these compatibility scenarios: **1. Linux Provisioning Script Changes** + - **Context**: Scripts in `parts/linux/cloud-init/artifacts/` run during critical VM bootstrap and are used in both: - VHD build (uploaded via packer configs in `vhdbuilder/packer/*.json`) - VM provisioning (CSE - embedded in Go service via `pkg/agent/const.go`) @@ -118,6 +117,7 @@ Analyze PRs for these compatibility scenarios: - **ANC hotfix entry removal**: If a PR removes or modifies the `hotfix-scripts: auto-generated` block in `parts/linux/cloud-init/nodecustomdata.yml`, or resets `parts/linux/cloud-init/artifacts/aks-node-controller-hotfix.json` to `{}`, **always confirm with the PR owner** that all affected VHDs have been republished with the fix baked in or are out of the 6-month support window. Premature removal means nodes provisioned via scale-up on the old buggy VHD will no longer receive the hotfix. These files are auto-generated by `hotfix/hotfix_generate.py` (via the `hotfix-generate` GH Action) — see that script for how `version`/`scripts_version` are computed. **2. Windows Bidirectional Compatibility** + - **Context**: Windows VHD and CSE scripts release on different cadences with no guaranteed order - **What to check**: Changes to `staging/cse/windows/` (CSE scripts) or `vhdbuilder/packer/windows/` (VHD scripts) - **Breaking signals**: @@ -127,6 +127,7 @@ Analyze PRs for these compatibility scenarios: - Removing PowerShell functions or cmdlets that the other component might call **3. aks-node-controller Migration (Dual-Mode Support)** + - **Context**: Transitioning from uploading scripts during both VHD build and CSE to only uploading aks-node-controller during VHD build - **What to check**: Any changes must work in BOTH deployment modes - **Breaking signals**: @@ -136,7 +137,8 @@ Analyze PRs for these compatibility scenarios: - Hardcoded paths that differ between deployment modes **4. PIS / VHD Caching — basePrep vs nodePrep split (Windows + Linux)** -- **Context**: PIS bakes a VHD from a temporary VM, then boots many real nodes from it. Same model in Windows `parts/windows/kuberneteswindowssetup.ps1` (`BasePrep`/`NodePrep`) and Linux `parts/linux/cloud-init/artifacts/cse_main.sh` (`basePrep`/`nodePrep`): + +- **Context**: PIS bakes a VHD from a temporary VM, then boots many real nodes from it. Same model in Windows `parts/windows/kuberneteswindowssetup.ps1.template` (`BasePrep`/`NodePrep`) and Linux `parts/linux/cloud-init/artifacts/cse_main.sh` (`basePrep`/`nodePrep`): - `basePrep` — gated only by the `base_prep.complete` marker (`C:\AzureData\` Windows, `/opt/azure/containers/` Linux). **Skipped on PIS real nodes** because the marker is baked into the VHD. - `nodePrep` — runs whenever `PreProvisionOnly` is false (every real node); skipped only on the bake VM. - The bake run sets `PreProvisionOnly=true` (`{{GetPreProvisionOnly}}`) and writes the marker after `basePrep` succeeds (Windows `finally`; Linux `cse_start.sh`). @@ -150,6 +152,7 @@ Analyze PRs for these compatibility scenarios: - **Don't flag**: plain variable reads (live on the real node from CustomData); cached binaries/packages/images; the `base_prep.complete` marker; cluster-wide non-secrets (CA cert, apiserver FQDN, service CIDR); pre-existing `basePrep` writes unless the PR newly depends on them. **5. Cross-OS Compatibility** + - **What to check**: Changes work on Ubuntu, Azure Linux/Mariner, and Windows - **Breaking signals**: - Linux commands that don't work on both Ubuntu and Azure Linux/Mariner @@ -158,6 +161,7 @@ Analyze PRs for these compatibility scenarios: - Systemd differences between distributions **6. Package/Dependency Update PRs (Renovate)** + - **Context**: Renovate bot automatically creates PRs to update component versions in `parts/common/components.json`. These components are cached on VHDs during build and directly affect node stability, GPU workloads, networking, and security. Updated packages are downloaded from `packages.aks.azure.com` or upstream registries during VHD build. - **What to check**: Every version bump—even patch versions—can introduce regressions that affect production nodes. - **Analysis steps for every package update PR**: @@ -181,6 +185,7 @@ Analyze PRs for these compatibility scenarios: - **Review output for package update PRs must include a detailed version diff analysis**: **Header:** + ``` ## Package Update Analysis: **Version change**: X.Y.Z → A.B.C ( update) @@ -215,6 +220,7 @@ Analyze PRs for these compatibility scenarios: **If upstream changelog is unavailable**, explicitly state: _"Upstream changelog not found for this version range. Manual testing recommended before merge."_ **Overall risk assessment:** + ``` ### Overall Risk: 🟢 Low / 🟡 Medium / 🔴 High **Justification**: <1-2 sentence summary of why this risk level was chosen> @@ -222,6 +228,7 @@ Analyze PRs for these compatibility scenarios: ``` **Example** (for a PR like dcgm-exporter 4.7.1 → 4.8.0): + ``` ## Package Update Analysis: dcgm-exporter **Version change**: 4.7.1 → 4.8.0 (minor update) @@ -244,6 +251,7 @@ Analyze PRs for these compatibility scenarios: ### Analysis Approach **Dynamic Dependency Tracing**: + 1. For each changed file, identify what depends on it 2. Follow `source` statements in bash scripts to trace dependency chains 3. Check for function calls, variable references across files @@ -255,10 +263,12 @@ Analyze PRs for these compatibility scenarios: - Flag downloads from unauthorized sources (only packages.aks.azure.com and sources in components.json allowed) **Historical Context**: + - Look for related changes that previously caused issues - Identify patterns of fragile areas that break frequently **Test Coverage Assessment**: + - Note if changed code has e2e test coverage - Flag changes to untested areas as higher risk - Mention if new behavior lacks corresponding test additions @@ -268,6 +278,7 @@ Analyze PRs for these compatibility scenarios: Provide targeted inline comments on specific lines where you detect issues: **For each breaking change or risk:** + - Comment directly on the problematic line or code block - Explain why this is risky (e.g., "This removes function X which may be called by VHDs built in the last 6 months") - Suggest specific mitigations or alternatives @@ -296,9 +307,14 @@ Provide targeted inline comments on specific lines where you detect issues: ### Review Philosophy Think like an experienced reviewer who "eyeballs" PRs for subtle risks. Look beyond pattern matching: + - Understand the architecture and how components interact - Consider timing of releases and deployment sequences - Reason about implicit dependencies and assumptions - Flag changes that "feel risky" even without obvious red flags - Balance thoroughness with actionable feedback -- Focus on high-impact issues that could break production VM provisioning \ No newline at end of file +- Focus on high-impact issues that could break production VM provisioning +- Reason about implicit dependencies and assumptions +- Flag changes that "feel risky" even without obvious red flags +- Balance thoroughness with actionable feedback +- Focus on high-impact issues that could break production VM provisioning diff --git a/parts/windows/kuberneteswindowssetup.ps1 b/parts/windows/kuberneteswindowssetup.ps1.template similarity index 100% rename from parts/windows/kuberneteswindowssetup.ps1 rename to parts/windows/kuberneteswindowssetup.ps1.template diff --git a/parts/windows/windowscsehelper.ps1 b/parts/windows/windowscsehelper.ps1 index 2d54c6f8d5e..303d331c3ac 100644 --- a/parts/windows/windowscsehelper.ps1 +++ b/parts/windows/windowscsehelper.ps1 @@ -4,7 +4,7 @@ # Define all exit codes in Windows CSE # It must match `[A-Z_]+` $global:WINDOWS_CSE_SUCCESS=0 -$global:WINDOWS_CSE_ERROR_UNKNOWN=1 # For unexpected error caught by the catch block in kuberneteswindowssetup.ps1 +$global:WINDOWS_CSE_ERROR_UNKNOWN=1 # For unexpected error caught by the catch block in kuberneteswindowssetup.ps1.template $global:WINDOWS_CSE_ERROR_DOWNLOAD_FILE_WITH_RETRY=2 $global:WINDOWS_CSE_ERROR_INVOKE_EXECUTABLE=3 $global:WINDOWS_CSE_ERROR_FILE_NOT_EXIST=4 diff --git a/pkg/agent/const.go b/pkg/agent/const.go index 19d1043aa15..24511398d11 100644 --- a/pkg/agent/const.go +++ b/pkg/agent/const.go @@ -30,7 +30,7 @@ const ( // kubernetesWindowsAgentCSECommandPS1 privides the command of Windows CSE. kubernetesWindowsAgentCSECommandPS1 = "windows/csecmd.ps1" // kubernetesWindowsAgentCustomDataPS1 is used for generating the customdata of Windows VM. - kubernetesWindowsAgentCustomDataPS1 = "windows/kuberneteswindowssetup.ps1" + kubernetesWindowsAgentCustomDataPS1 = "windows/kuberneteswindowssetup.ps1.template" /* Windows CSE helper scripts. These should all be listed in baker.go:func GetKubernetesWindowsAgentFunctions. */ kubernetesWindowsCSEHelperPS1 = "windows/windowscsehelper.ps1" diff --git a/staging/cse/windows/README b/staging/cse/windows/README index 419c8c03986..82f932e5169 100644 --- a/staging/cse/windows/README +++ b/staging/cse/windows/README @@ -19,7 +19,7 @@ ```bash branchName="master" -currentCseVersion="v0.0.51" # `WindowsCSEScriptsPackage` defined in `parts/windows/kuberneteswindowssetup.ps1` +currentCseVersion="v0.0.51" # `WindowsCSEScriptsPackage` defined in `parts/windows/kuberneteswindowssetup.ps1.template` testCseVersion="v0.0.51.0" # Test package name. NOTE: Please do not use the official package format and earlier used version. url="https://raw.githubusercontent.com/Azure/AgentBaker/$branchName/staging/cse/windows" From ecf33a9a4cdb64e40a6b8fece5d339f54f2a1f6d Mon Sep 17 00:00:00 2001 From: Nishchay Date: Thu, 6 Aug 2026 21:09:07 -0700 Subject: [PATCH 13/38] fix: enable ip connect in bastion, something changed recently causing us to need it (#9149) --- e2e/shared_infra.go | 1 + 1 file changed, 1 insertion(+) diff --git a/e2e/shared_infra.go b/e2e/shared_infra.go index 0357613b700..cd0f5f78ac4 100644 --- a/e2e/shared_infra.go +++ b/e2e/shared_infra.go @@ -331,6 +331,7 @@ func ensureSharedBastion(ctx context.Context, rg, location string) (string, erro }, Properties: &armnetwork.BastionHostPropertiesFormat{ EnableTunneling: to.Ptr(true), + EnableIPConnect: to.Ptr(true), IPConfigurations: []*armnetwork.BastionHostIPConfiguration{ { Name: to.Ptr("bastion-ipcfg"), From 832839aacf4cf98dcc0de4ec811a920a7f6fd9bc Mon Sep 17 00:00:00 2001 From: Nishchay Date: Thu, 6 Aug 2026 21:48:59 -0700 Subject: [PATCH 14/38] fix: add renovate support for 2604 (#9158) --- .github/renovate.json | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/.github/renovate.json b/.github/renovate.json index d8f868dbdad..b958a649df2 100644 --- a/.github/renovate.json +++ b/.github/renovate.json @@ -58,7 +58,8 @@ "custom.deb2004", "custom.deb2204", "custom.deb2404", - "custom.deb2404-test" + "custom.deb2404-test", + "custom.deb2604" ], "matchUpdateTypes": [ "patch" @@ -845,6 +846,20 @@ "versioningTemplate": "deb", "autoReplaceStringTemplate": "\"renovateTag\": \"name={{{packageName}}}, repository=test, os=ubuntu, release=24.04\",\n \"latestVersion\": \"{{{newValue}}}\"{{#if depType}},\n \"previousLatestVersion\": \"{{{currentValue}}}\"{{/if}}" }, + { + "customType": "regex", + "description": "auto update packages for OS ubuntu 26.04 in components.json", + "managerFilePatterns": [ + "/parts/common/components.json/" + ], + "matchStringsStrategy": "any", + "matchStrings": [ + "\"renovateTag\":\\s*\"name=(?[^\"]+), repository=production, os=ubuntu, release=26\\.04\",\\s*\"latestVersion\":\\s*\"(?[^\"]+)\"(?:[^}]*\"previousLatestVersion\":\\s*\"(?[^\"]+)\")?" + ], + "datasourceTemplate": "custom.deb2604", + "versioningTemplate": "deb", + "autoReplaceStringTemplate": "\"renovateTag\": \"name={{{packageName}}}, repository=production, os=ubuntu, release=26.04\",\n \"latestVersion\": \"{{{newValue}}}\"{{#if depType}},\n \"previousLatestVersion\": \"{{{currentValue}}}\"{{/if}}" + }, { "customType": "regex", "description": "auto update packages for OS Mariner 2.0 in components.json", @@ -996,6 +1011,13 @@ "transformTemplates": [ "{\"releases\": $map(($index := releases#$i[version=\"Package: {{packageName}}\"].$i; $map($index, function($i) { $substringAfter(releases[$i + 1].version, \"Version: \") })), function($v) { {\"version\": $v} })[]}" ] + }, + "deb2604": { + "defaultRegistryUrlTemplate": "https://packages.microsoft.com/ubuntu/26.04/prod/dists/resolute/main/binary-amd64/Packages", + "format": "plain", + "transformTemplates": [ + "{\"releases\": $map(($index := releases#$i[version=\"Package: {{packageName}}\"].$i; $map($index, function($i) { $substringAfter(releases[$i + 1].version, \"Version: \") })), function($v) { {\"version\": $v} })[]}" + ] } } } From 4b141b31c27f3d5c0e0a8c899927fd01b468b8be Mon Sep 17 00:00:00 2001 From: Andy Zhang Date: Fri, 7 Aug 2026 15:39:40 +0800 Subject: [PATCH 15/38] chore: upgrade Azure Disk CSI driver versions (#9157) --- parts/common/components.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/parts/common/components.json b/parts/common/components.json index a7bdb04b5ea..246335d069c 100644 --- a/parts/common/components.json +++ b/parts/common/components.json @@ -506,23 +506,23 @@ "multiArchVersionsV2": [ { "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes-csi/azuredisk-csi", - "latestVersion": "v1.34.4" + "latestVersion": "v1.34.5" }, { "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes-csi/azuredisk-csi", - "latestVersion": "v1.33.10" + "latestVersion": "v1.33.11" } ], "windowsVersions": [ { "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes-csi/azuredisk-csi", - "latestVersion": "v1.34.4-windows-hp", - "previousLatestVersion": "v1.34.3-windows-hp" + "latestVersion": "v1.34.5-windows-hp", + "previousLatestVersion": "v1.34.4-windows-hp" }, { "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes-csi/azuredisk-csi", - "latestVersion": "v1.33.10-windows-hp", - "previousLatestVersion": "v1.33.9-windows-hp" + "latestVersion": "v1.33.11-windows-hp", + "previousLatestVersion": "v1.33.10-windows-hp" } ] }, From ec99e53ba3ca1f07ae43588c7b8e425c4f341206 Mon Sep 17 00:00:00 2001 From: Martin Heberling Date: Sat, 8 Aug 2026 20:34:38 -0700 Subject: [PATCH 16/38] test(e2e): add Kata Containers E2E scenario for AzureLinux V3 (#9142) --- e2e/config/vhd.go | 12 ++ e2e/node_config.go | 6 + e2e/scenario_test.go | 49 ++++++++ e2e/types.go | 1 + e2e/validators.go | 15 +++ e2e/validators_kata.go | 268 +++++++++++++++++++++++++++++++++++++++++ 6 files changed, 351 insertions(+) create mode 100644 e2e/validators_kata.go diff --git a/e2e/config/vhd.go b/e2e/config/vhd.go index a79b7257e97..38a1f40df20 100644 --- a/e2e/config/vhd.go +++ b/e2e/config/vhd.go @@ -128,6 +128,18 @@ var ( Gallery: imageGalleryLinux, } + // VHDAzureLinuxV3Gen2Kata is the AzureLinux V3 Gen2 VHD built with FEATURE_FLAGS=kata. + // The image definition name mirrors the SIG_IMAGE_NAME produced by the VHD builder for + // OS_VERSION=V3kata + HYPERV_GENERATION=V2 (SKU_NAME=V3katagen2, prefixed with "AzureLinux"). + // See .pipelines/.vsts-vhd-builder-release.yaml (buildAzureLinuxV3gen2kata) and + // vhdbuilder/packer/produce-packer-settings-functions.sh (ensure_sig_image_name_linux). + VHDAzureLinuxV3Gen2Kata = &Image{ + Name: "AzureLinuxV3katagen2", + OS: OSAzureLinux, + Arch: "amd64", + Distro: datamodel.AKSAzureLinuxV3Gen2Kata, + Gallery: imageGalleryLinux, + } VHDAzureLinux3OSGuard = &Image{ Name: "AzureLinuxOSGuardOSGuardV3gen2fipsTL", OS: OSAzureLinux, diff --git a/e2e/node_config.go b/e2e/node_config.go index 52cbef44d28..bc1aaa6b817 100644 --- a/e2e/node_config.go +++ b/e2e/node_config.go @@ -212,6 +212,12 @@ func nbcToAKSNodeConfigV1(nbc *datamodel.NodeBootstrappingConfiguration) (*aksno DisableCustomData: true, LinuxAdminUsername: "azureuser", VmSize: config.Config.DefaultVMSKU, + // The scriptless/aks-node-controller path gates its Kata containerd config blocks on + // this field alone, whereas the NBC/baker path derives Kata from the agent pool distro + // (see Distro.IsKataDistro and the IsKata template func in pkg/agent/baker.go). Without + // this mapping a scriptless scenario running on a Kata VHD would silently get a Kata + // image with a non-Kata containerd config. + IsKata: nbc.AgentPoolProfile.Distro.IsKataDistro(), ClusterConfig: &aksnodeconfigv1.ClusterConfig{ Location: nbc.ContainerService.Location, ResourceGroup: nbc.ResourceGroupName, diff --git a/e2e/scenario_test.go b/e2e/scenario_test.go index f1b660dc4f6..2792f170b40 100644 --- a/e2e/scenario_test.go +++ b/e2e/scenario_test.go @@ -411,6 +411,55 @@ func Test_AzureLinuxV3(t *testing.T) { }) } +// Test_AzureLinuxV3Gen2Kata verifies that AgentBaker correctly bootstraps a Kata-enabled node. +// +// Kata Containers is a runtime, so the thing that can silently break is the containerd +// configuration: pkg/agent/baker.go only emits the `kata` runtime handler blocks +// when the agent pool's Distro satisfies Distro.IsKataDistro(). Selecting a Kata VHD here flows +// through e2e/node_config.go -> AgentPoolProfile.Distro -> the IsKata template func, so this +// scenario exercises that whole path against a real node. +// +// The scenario asserts three increasingly strong properties: +// 1. the rendered /etc/containerd/config.toml contains the Kata runtime handlers, +// 2. containerd actually parsed and loaded them (no warnings, handlers in `config dump`), +// 3. for every handler in kataRuntimeHandlers, a pod scheduled via a Kata RuntimeClass runs +// and is genuinely VM-isolated. +func Test_AzureLinuxV3Gen2Kata(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that an AzureLinuxV3 Gen2 Kata node is bootstrapped with working kata containerd runtime handlers, and can run VM-isolated pods via Kata RuntimeClasses", + Tags: Tags{ + Kata: true, + }, + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDAzureLinuxV3Gen2Kata, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + // Kata launches each pod inside its own VM, which requires nested virtualization + // and more headroom than the 2-vCPU e2e default (config.Config.DefaultVMSKU). + nbc.ContainerService.Properties.AgentPoolProfiles[0].VMSize = kataVMSize + nbc.AgentPoolProfile.VMSize = kataVMSize + // Leave unattended upgrades on so that CSE's kata-specific opt-out branch is + // actually exercised, which ValidateKataHostReadiness asserts. + nbc.DisableUnattendedUpgrades = false + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.SKU.Name = to.Ptr(kataVMSize) + }, + Validator: func(ctx context.Context, s *Scenario) { + ValidateKataContainerdConfig(ctx, s) + ValidateKataContainerdConfigDump(ctx, s) + ValidateKataHostReadiness(ctx, s) + for _, handler := range kataRuntimeHandlers { + ValidateKataPodIsIsolated(ctx, s, handler) + } + }, + }, + }) +} + +// kataVMSize is a nested-virtualization capable SKU with enough capacity to host a Kata guest VM. +const kataVMSize = "Standard_D4ds_v5" + func Test_AzureLinuxV3_CustomCA(t *testing.T) { RunScenario(t, &Scenario{ Description: "Tests that an AzureLinuxV3 node can be properly bootstrapped with custom ca", diff --git a/e2e/types.go b/e2e/types.go index 50223ad8c26..99a7e0ac50f 100644 --- a/e2e/types.go +++ b/e2e/types.go @@ -30,6 +30,7 @@ type Tags struct { NonAnonymousACR bool GPU bool WASM bool + Kata bool BootstrapTokenFallback bool KubeletCustomConfig bool Scriptless bool diff --git a/e2e/validators.go b/e2e/validators.go index cc5d5c2d629..656ace66591 100644 --- a/e2e/validators.go +++ b/e2e/validators.go @@ -1264,6 +1264,21 @@ func ValidateContainerd2Properties(ctx context.Context, s *Scenario, versions [] ValidateInstalledPackageVersion(ctx, s, "moby-containerd", versions[0]) + // TODO: this assertion never actually runs and always passes vacuously. + // + // execOnVMForScenarioOnUnprivilegedPod executes inside the "debugnonhost" daemonset pod, + // which runs a bare mcr.microsoft.com/cbl-mariner/base/core:2.0 image with no volume mounts + // (see daemonsetDebug in e2e/kube.go). The host filesystem is not mounted into it, so the + // containerd binary is unreachable and the command exits 127 with an empty stdout and + // "command not found" on stderr. attemptExecOnPod treats a non-zero exit as a successful + // exec, so no error surfaces, and the NotContains check below then trivially passes against + // an empty string. + // + // Two changes are needed: run this on the node via execScriptOnVMForScenarioValidateExitCode + // (as ValidateKataContainerdConfigDump in validators_kata.go now does), and check stderr as + // well as stdout, since containerd logs its warnings to stderr. Fixing it is likely to + // surface real warnings at the 11 call sites that use this validator, so it is left as a + // follow-up rather than folded into an unrelated change. execResult := execOnVMForScenarioOnUnprivilegedPod(ctx, s, "containerd config dump ") // validate containerd config dump has no warnings require.NotContains(s.T, execResult.stdout, "level=warning", "do not expect warning message when converting config file %", execResult.stdout) diff --git a/e2e/validators_kata.go b/e2e/validators_kata.go new file mode 100644 index 00000000000..44cf1920b83 --- /dev/null +++ b/e2e/validators_kata.go @@ -0,0 +1,268 @@ +package e2e + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + nodev1 "k8s.io/api/node/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // kataRuntimeHandler is the containerd runtime handler name for standard Kata Containers, + // emitted by the IsKata block of the containerd config templates in pkg/agent/baker.go. + kataRuntimeHandler = "kata" + + // kataConfigPath is the Kata configuration file referenced by the "kata" runtime handler's + // options.ConfigPath in the rendered containerd config. + kataConfigPath = "/usr/share/defaults/kata-containers/configuration.toml" + + // containerdConfigPath is where CSE / aks-node-controller writes the rendered containerd config. + containerdConfigPath = "/etc/containerd/config.toml" +) + +// kataRuntimeHandlers lists every Kata containerd runtime handler that this scenario expects to +// be configured and usable on the node. Each one is asserted in the effective containerd config +// and independently exercised by ValidateKataPodIsIsolated, so covering an additional handler is +// a one-line change here plus a call in the scenario. +// +// Note that "kata-cc" (confidential containers) is intentionally absent: its handler block is +// templated for all Kata VHDs, but it targets a different VHD than regular Kata, so this image +// cannot actually run it. +var kataRuntimeHandlers = []string{kataRuntimeHandler} + +// ValidateKataContainerdConfig asserts that AgentBaker rendered a containerd configuration +// containing the Kata runtime handlers on a Kata-enabled VHD. +// +// This is the core regression check for the IsKata blocks of the containerd config templates in +// pkg/agent/baker.go. Note that AgentPoolProfile.IsContainerdV2Distro() returns false for every +// Kata distro (pkg/agent/datamodel/types.go), so Kata nodes are always rendered from +// containerdV1ConfigTemplate / containerdV1NoGPUConfigTemplate regardless of the underlying OS. +// The assertions below therefore target the containerd 1.x plugin paths that those templates +// emit. If Kata is ever promoted to the V2 templates, this validator should fail loudly rather +// than silently pass, which is why the plugin paths are asserted explicitly. +func ValidateKataContainerdConfig(ctx context.Context, s *Scenario) { + s.T.Helper() + + require.True(s.T, s.VHD.Distro.IsKataDistro(), + "ValidateKataContainerdConfig requires a Kata distro, got %q", s.VHD.Distro) + + // The standard "kata" runtime handler, backed by the kata v2 shim. + ValidateFileHasContent(ctx, s, containerdConfigPath, `[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata]`) + ValidateFileHasContent(ctx, s, containerdConfigPath, `runtime_type = "io.containerd.kata.v2"`) + ValidateFileHasContent(ctx, s, containerdConfigPath, kataConfigPath) + + // Kata relies on snapshot annotations being forwarded to the snapshotter; the template sets + // this explicitly under IsKata and disabling it breaks image pulling for Kata pods. + ValidateFileHasContent(ctx, s, containerdConfigPath, "disable_snapshot_annotations = false") +} + +// ValidateKataContainerdConfigDump asserts that containerd itself accepted the rendered +// configuration and actually loaded the Kata runtime handlers. +// +// Checking the file alone is not enough. Kata VHDs ship their own containerd build - CSE skips +// installing one (see the "azurelinuxkata" entries in parts/common/components.json) - so the +// containerd major version on the node is decided by the image, not by AgentBaker, while the +// template AgentBaker renders is decided by the distro (IsContainerdV2Distro short-circuits to +// the v1 template for every Kata distro). The two can therefore disagree: AzureLinux V3 Kata +// currently boots containerd 2.x while being handed a containerd 1.x style config. +// +// That combination happens to work today because containerd 2.x migrates the legacy +// "io.containerd.grpc.v1.cri" runtime handlers onto the current +// "io.containerd.cri.v1.runtime" paths, but nothing guarantees it keeps doing so. This +// validator pins the property we actually care about: after containerd has parsed the config, +// the Kata handlers are present in the effective configuration and containerd raised no +// warnings while getting there. +func ValidateKataContainerdConfigDump(ctx context.Context, s *Scenario) { + s.T.Helper() + + // This must run on the node itself, not in a debug pod. The "debugnonhost" daemonset pods + // used by execOnVMForScenarioOnUnprivilegedPod run a bare CBL-Mariner base image with no + // volume mounts, so the host's containerd binary is not reachable from them and the command + // would simply exit 127. + execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, "sudo containerd config dump", 0, + "unable to dump the effective containerd config on the node") + + // The effective config is printed on stdout, but containerd logs diagnostics (including the + // "level=warning" lines we care about) on stderr, so both streams have to be inspected. + dump := execResult.stdout + diagnostics := execResult.stdout + "\n" + execResult.stderr + + // "containerd config dump" re-serializes the config and quotes TOML strings with single + // quotes, whereas the config file AgentBaker generates uses double quotes. Normalize so the + // assertions below can be written the way the config file reads. + normalizedDump := strings.ReplaceAll(dump, "'", `"`) + + // The effective config must expose every Kata runtime handler we expect. Note the trailing + // "]": without it a handler name would also match longer handlers sharing its prefix (e.g. + // "runtimes.kata" matching "runtimes.kata-preview") and pass even if the handler itself + // were missing. + for _, handler := range kataRuntimeHandlers { + assert.Contains(s.T, normalizedDump, `runtimes.`+handler+`]`, + "expected the %q runtime handler in the effective containerd config.\nDump:\n%s", handler, dump) + } + assert.Contains(s.T, normalizedDump, `runtime_type = "io.containerd.kata.v2"`, + "expected the kata v2 shim runtime_type in the effective containerd config.\nDump:\n%s", dump) + + // A warning here means containerd did not fully understand the config we generated, e.g. it + // had to fall back on deprecated handling for the legacy plugin paths the Kata templates use. + assert.NotContains(s.T, diagnostics, "level=warning", + "containerd reported warnings while parsing the AgentBaker-generated config.\nstdout:\n%s\nstderr:\n%s", + execResult.stdout, execResult.stderr) +} + +// ValidateKataHostReadiness asserts the host-side prerequisites that the Kata VHD is expected to +// ship and that the containerd config references. Without these, the containerd config would be +// syntactically valid but the kata shim would fail at pod sandbox creation time. +func ValidateKataHostReadiness(ctx context.Context, s *Scenario) { + s.T.Helper() + + // The kata shim binary that runtime_type = "io.containerd.kata.v2" resolves to. + execScriptOnVMForScenarioValidateExitCode(ctx, s, + "command -v containerd-shim-kata-v2", 0, "containerd-shim-kata-v2 is not present on the Kata VHD") + + // The Kata configuration file referenced by options.ConfigPath in the containerd config. + ValidateFileExists(ctx, s, kataConfigPath) + + // Kata VHDs deliberately opt out of automatic package updates even when unattended upgrades + // are enabled, because kata packages must be updated as a unit (including the kernel, which + // requires a reboot). See the IS_KATA branch in parts/linux/cloud-init/artifacts/cse_main.sh. + // The scenario leaves unattended upgrades enabled so this branch is genuinely exercised. + execScriptOnVMForScenarioValidateExitCode(ctx, s, + "systemctl is-enabled dnf-automatic-install.timer", 1, + "dnf-automatic-install.timer must not be enabled on Kata VHDs: kata packages have to be updated as a unit via image updates") +} + +// ValidateKataPodIsIsolated creates a RuntimeClass bound to the given Kata runtime handler, +// schedules a pod against it on the node under test, and asserts the pod is genuinely running +// inside a Kata VM. +// +// This is the end-to-end proof that the containerd config AgentBaker generated is not merely +// syntactically present but actually usable: if the runtime handler were missing or +// misconfigured, the kubelet would reject the pod with "RuntimeHandler not supported" and the +// pod would never reach Running. +// +// Isolation itself is asserted by comparing kernel releases. A Kata pod boots its own guest +// kernel, so it must report a different `uname -r` than the host; a matching value would mean +// the pod silently fell back to the shared-kernel runc runtime. +// +// The RuntimeClass is pinned to this scenario's node via Scheduling.NodeSelector so it cannot +// interfere with other scenarios running in parallel against the same cluster, and is named +// after the handler so that several handlers can be validated on one node. +func ValidateKataPodIsIsolated(ctx context.Context, s *Scenario, handler string) { + s.T.Helper() + + hostKernel := strings.TrimSpace( + execScriptOnVMForScenarioValidateExitCode(ctx, s, "uname -r", 0, "unable to read host kernel release").stdout) + require.NotEmpty(s.T, hostKernel, "host kernel release was empty") + + runtimeClassName := createKataRuntimeClass(ctx, s, handler) + pod := createKataPod(ctx, s, runtimeClassName, handler) + + execResult, err := execOnPod(ctx, s.Runtime.Kube, pod.Namespace, pod.Name, []string{"uname", "-r"}) + require.NoErrorf(s.T, err, "failed to exec in kata pod %q", pod.Name) + guestKernel := strings.TrimSpace(execResult.stdout) + require.NotEmpty(s.T, guestKernel, "kata guest kernel release was empty") + + s.T.Logf("host kernel: %q, kata guest kernel: %q", hostKernel, guestKernel) + assert.NotEqual(s.T, hostKernel, guestKernel, + "pod running under the %q RuntimeClass reported the same kernel release as the host, "+ + "which means it was not launched inside a Kata VM", handler) +} + +// createKataRuntimeClass creates a RuntimeClass for the given handler scoped to the scenario's +// node and registers its cleanup. It returns the RuntimeClass name. +func createKataRuntimeClass(ctx context.Context, s *Scenario, handler string) string { + s.T.Helper() + + kube := s.Runtime.Kube + name := truncateKataResourceName(fmt.Sprintf("%s-%s", handler, s.Runtime.VM.KubeName)) + + runtimeClass := &nodev1.RuntimeClass{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Handler: handler, + Scheduling: &nodev1.Scheduling{ + NodeSelector: map[string]string{"kubernetes.io/hostname": s.Runtime.VM.KubeName}, + }, + } + + _, err := kube.Typed.NodeV1().RuntimeClasses().Create(ctx, runtimeClass, metav1.CreateOptions{}) + require.NoErrorf(s.T, err, "failed to create RuntimeClass %q for handler %q", name, handler) + + s.T.Cleanup(func() { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + if err := kube.Typed.NodeV1().RuntimeClasses().Delete(cleanupCtx, name, metav1.DeleteOptions{}); err != nil { + s.T.Logf("could not delete RuntimeClass %s: %v", name, err) + } + }) + + return name +} + +// createKataPod creates a long-lived pod bound to the given Kata RuntimeClass on the scenario's +// node, waits for it to reach Running, and registers its cleanup. Unlike ValidatePodRunning the +// pod is kept alive after this returns so callers can exec into it. +func createKataPod(ctx context.Context, s *Scenario, runtimeClassName, handler string) *corev1.Pod { + s.T.Helper() + + kube := s.Runtime.Kube + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: truncateKataResourceName(fmt.Sprintf("%s-%s-pod", s.Runtime.VM.KubeName, handler)), + Namespace: "default", + }, + Spec: corev1.PodSpec{ + RuntimeClassName: to.Ptr(runtimeClassName), + Containers: []corev1.Container{ + { + Name: "workload", + Image: "mcr.microsoft.com/cbl-mariner/busybox:2.0", + Command: []string{"sh", "-c"}, + Args: []string{"sleep 3600"}, + }, + }, + NodeSelector: map[string]string{ + "kubernetes.io/hostname": s.Runtime.VM.KubeName, + }, + }, + } + + s.T.Logf("creating pod %q under RuntimeClass %q", pod.Name, runtimeClassName) + _, err := kube.Typed.CoreV1().Pods(pod.Namespace).Create(ctx, pod, metav1.CreateOptions{}) + require.NoErrorf(s.T, err, "failed to create kata pod %q", pod.Name) + + s.T.Cleanup(func() { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + err := kube.Typed.CoreV1().Pods(pod.Namespace).Delete(cleanupCtx, pod.Name, metav1.DeleteOptions{ + GracePeriodSeconds: to.Ptr(int64(0)), + }) + if err != nil { + s.T.Logf("could not delete pod %s: %v", pod.Name, err) + } + }) + + running, err := kube.WaitUntilPodRunning(ctx, pod.Namespace, "", "metadata.name="+pod.Name) + require.NoErrorf(s.T, err, + "kata pod %q never reached Running. This usually means containerd did not register the %q "+ + "runtime handler from the AgentBaker-generated config", pod.Name, handler) + + return running +} + +// truncateKataResourceName keeps generated Kubernetes object names within the 63 character +// DNS-1123 label limit. +func truncateKataResourceName(name string) string { + const maxLen = 63 + if len(name) <= maxLen { + return name + } + return strings.TrimRight(name[:maxLen], "-") +} From 457167bc79a990c85388f8788c03c3543f37b4ad Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 10 Aug 2026 12:04:33 +1000 Subject: [PATCH 17/38] chore(deps): update acr-credential-provider (patch) (#9130) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- parts/common/components.json | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/parts/common/components.json b/parts/common/components.json index 246335d069c..d5659e61442 100644 --- a/parts/common/components.json +++ b/parts/common/components.json @@ -1822,8 +1822,8 @@ { "k8sVersion": "1.36", "renovateTag": "name=azure-acr-credential-provider, repository=production, os=ubuntu, release=26.04", - "latestVersion": "1.36.3-ubuntu26.04u3", - "previousLatestVersion": "1.36.2-ubuntu26.04u3" + "latestVersion": "1.36.4-ubuntu26.04u1", + "previousLatestVersion": "1.36.3-ubuntu26.04u3" } ] }, @@ -1888,20 +1888,20 @@ { "k8sVersion": "1.34", "renovateTag": "RPM_registry=https://packages.microsoft.com/azurelinux/3.0/prod/cloud-native/x86_64/repodata, name=azure-acr-credential-provider, os=azurelinux, release=3.0", - "latestVersion": "1.34.12-2.azl3", - "previousLatestVersion": "1.34.11-2.azl3" + "latestVersion": "1.34.13-1.azl3", + "previousLatestVersion": "1.34.12-2.azl3" }, { "k8sVersion": "1.35", "renovateTag": "RPM_registry=https://packages.microsoft.com/azurelinux/3.0/prod/cloud-native/x86_64/repodata, name=azure-acr-credential-provider, os=azurelinux, release=3.0", - "latestVersion": "1.35.7-2.azl3", - "previousLatestVersion": "1.35.6-2.azl3" + "latestVersion": "1.35.8-1.azl3", + "previousLatestVersion": "1.35.7-2.azl3" }, { "k8sVersion": "1.36", "renovateTag": "RPM_registry=https://packages.microsoft.com/azurelinux/3.0/prod/cloud-native/x86_64/repodata, name=azure-acr-credential-provider, os=azurelinux, release=3.0", - "latestVersion": "1.36.3-3.azl3", - "previousLatestVersion": "1.36.2-3.azl3" + "latestVersion": "1.36.4-1.azl3", + "previousLatestVersion": "1.36.3-3.azl3" } ] } From 2a1092fa2c08b9bebcadcf68bf3b85b460fbd316 Mon Sep 17 00:00:00 2001 From: Tim Wright Date: Mon, 10 Aug 2026 16:41:28 +1200 Subject: [PATCH 18/38] fix(windows): improve CSE error handling and consolidate function script sourcing (#9160) --- .../kuberneteswindowssetup.ps1.template | 507 +++++++++--------- parts/windows/windowscsehelper.ps1 | 356 ++++++------ staging/cse/windows/all.ps1 | 12 + 3 files changed, 452 insertions(+), 423 deletions(-) create mode 100644 staging/cse/windows/all.ps1 diff --git a/parts/windows/kuberneteswindowssetup.ps1.template b/parts/windows/kuberneteswindowssetup.ps1.template index 014c77e61ae..fcedc292332 100644 --- a/parts/windows/kuberneteswindowssetup.ps1.template +++ b/parts/windows/kuberneteswindowssetup.ps1.template @@ -36,19 +36,20 @@ param( $CSEResultFilePath ) + # In an ideal world, all these values would be passed to this script in parameters. However, we don't live in an ideal world. # https://learn.microsoft.com/en-gb/troubleshoot/windows-client/shell-experience/command-line-string-limitation -$MasterIP = "{{ GetKubernetesEndpoint }}" -$KubeDnsServiceIp="{{ GetParameter "kubeDNSServiceIP" }}" -$MasterFQDNPrefix="{{ GetParameter "masterEndpointDNSNamePrefix" }}" -$Location="{{ GetVariable "location" }}" +$MasterIP="{{GetKubernetesEndpoint}}" +$KubeDnsServiceIp="{{GetParameter "kubeDNSServiceIP"}}" +$MasterFQDNPrefix="{{GetParameter "masterEndpointDNSNamePrefix"}}" +$Location="{{GetVariable "location"}}" {{if UserAssignedIDEnabled}} -$UserAssignedClientID="{{ GetVariable "userAssignedIdentityID" }}" -{{ end }} -$TargetEnvironment="{{ GetTargetEnvironment }}" -$ArmResourceEndpoint="{{ GetArmResourceEndpoint }}" -$AADClientId="{{ GetParameter "servicePrincipalClientId" }}" +$UserAssignedClientID="{{GetVariable "userAssignedIdentityID"}}" +{{end}} +$TargetEnvironment="{{GetTargetEnvironment}}" +$ArmResourceEndpoint="{{GetArmResourceEndpoint}}" +$AADClientId="{{GetParameter "servicePrincipalClientId"}}" $NetworkAPIVersion="2018-08-01" # Do not parse the start time from $LogFile to simplify the logic @@ -60,293 +61,301 @@ $global:ErrorMessage="" # passed as powershell parameters ## SSH public keys to add to authorized_keys -$global:SSHKeys = @( {{ GetSshPublicKeysPowerShell }} ) +$global:SSHKeys=@( {{GetSshPublicKeysPowerShell}} ) ## Certificates generated by aks-engine -$global:CACertificate = "{{GetParameter "caCertificate"}}" -$global:AgentCertificate = "{{GetParameter "clientCertificate"}}" +$global:CACertificate="{{GetParameter "caCertificate"}}" +$global:AgentCertificate="{{GetParameter "clientCertificate"}}" ## Download sources provided by aks-engine -$global:KubeBinariesPackageSASURL = "{{GetParameter "kubeBinariesSASURL"}}" -$global:WindowsKubeBinariesURL = "{{GetParameter "windowsKubeBinariesURL"}}" -$global:KubeBinariesVersion = "{{GetParameter "kubeBinariesVersion"}}" -$global:ContainerdUrl = "{{GetParameter "windowsContainerdURL"}}" -$global:ContainerdSdnPluginUrl = "{{GetParameter "windowsSdnPluginURL"}}" +$global:KubeBinariesPackageSASURL="{{GetParameter "kubeBinariesSASURL"}}" +$global:WindowsKubeBinariesURL="{{GetParameter "windowsKubeBinariesURL"}}" +$global:KubeBinariesVersion="{{GetParameter "kubeBinariesVersion"}}" +$global:ContainerdUrl="{{GetParameter "windowsContainerdURL"}}" +$global:ContainerdSdnPluginUrl="{{GetParameter "windowsSdnPluginURL"}}" ## Docker Version -$global:DockerVersion = "{{GetParameter "windowsDockerVersion"}}" +$global:DockerVersion="{{GetParameter "windowsDockerVersion"}}" ## ContainerD Usage -$global:DefaultContainerdWindowsSandboxIsolation = "{{GetParameter "defaultContainerdWindowsSandboxIsolation"}}" -$global:ContainerdWindowsRuntimeHandlers = "{{GetParameter "containerdWindowsRuntimeHandlers"}}" +$global:DefaultContainerdWindowsSandboxIsolation="{{GetParameter "defaultContainerdWindowsSandboxIsolation"}}" +$global:ContainerdWindowsRuntimeHandlers="{{GetParameter "containerdWindowsRuntimeHandlers"}}" ## VM configuration passed by Azure -$global:WindowsTelemetryGUID = "{{GetParameter "windowsTelemetryGUID"}}" +$global:WindowsTelemetryGUID="{{GetParameter "windowsTelemetryGUID"}}" {{if eq GetIdentitySystem "adfs"}} -$global:TenantId = "adfs" +$global:TenantId="adfs" {{else}} -$global:TenantId = "{{GetVariable "tenantID"}}" +$global:TenantId="{{GetVariable "tenantID"}}" {{end}} -$global:SubscriptionId = "{{GetVariable "subscriptionId"}}" -$global:ResourceGroup = "{{GetVariable "resourceGroup"}}" -$global:VmType = "{{GetVariable "vmType"}}" -$global:SubnetName = "{{GetVariable "subnetName"}}" +$global:SubscriptionId="{{GetVariable "subscriptionId"}}" +$global:ResourceGroup="{{GetVariable "resourceGroup"}}" +$global:VmType="{{GetVariable "vmType"}}" +$global:SubnetName="{{GetVariable "subnetName"}}" # NOTE: MasterSubnet is still referenced by `kubeletstart.ps1` and `windowsnodereset.ps1` # for case of Kubenet -$global:MasterSubnet = "" -$global:SecurityGroupName = "{{GetVariable "nsgName"}}" -$global:VNetName = "{{GetVariable "virtualNetworkName"}}" -$global:RouteTableName = "{{GetVariable "routeTableName"}}" -$global:PrimaryAvailabilitySetName = "{{GetVariable "primaryAvailabilitySetName"}}" -$global:PrimaryScaleSetName = "{{GetVariable "primaryScaleSetName"}}" - -$global:KubeClusterCIDR = "{{GetParameter "kubeClusterCidr"}}" -$global:KubeServiceCIDR = "{{GetParameter "kubeServiceCidr"}}" -$global:VNetCIDR = "{{GetParameter "vnetCidr"}}" +$global:MasterSubnet="" +$global:SecurityGroupName="{{GetVariable "nsgName"}}" +$global:VNetName="{{GetVariable "virtualNetworkName"}}" +$global:RouteTableName="{{GetVariable "routeTableName"}}" +$global:PrimaryAvailabilitySetName="{{GetVariable "primaryAvailabilitySetName"}}" +$global:PrimaryScaleSetName="{{GetVariable "primaryScaleSetName"}}" + +$global:KubeClusterCIDR="{{GetParameter "kubeClusterCidr"}}" +$global:KubeServiceCIDR="{{GetParameter "kubeServiceCidr"}}" +$global:VNetCIDR="{{GetParameter "vnetCidr"}}" {{if IsKubernetesVersionGe "1.16.0"}} -$global:KubeletNodeLabels = "{{GetAgentKubernetesLabels . }}" +$global:KubeletNodeLabels="{{GetAgentKubernetesLabels .}}" {{else}} -$global:KubeletNodeLabels = "{{GetAgentKubernetesLabelsDeprecated . }}" +$global:KubeletNodeLabels="{{GetAgentKubernetesLabelsDeprecated .}}" {{end}} -$global:KubeletConfigArgs = @( {{GetKubeletConfigKeyValsPsh}} ) -$global:KubeproxyConfigArgs = @( {{GetKubeproxyConfigKeyValsPsh}} ) +$global:KubeletConfigArgs=@( {{GetKubeletConfigKeyValsPsh}} ) +$global:KubeproxyConfigArgs=@( {{GetKubeproxyConfigKeyValsPsh}} ) -$global:KubeproxyFeatureGates = @( {{GetKubeProxyFeatureGatesPsh}} ) +$global:KubeproxyFeatureGates=@( {{GetKubeProxyFeatureGatesPsh}} ) -$global:UseManagedIdentityExtension = "{{GetVariable "useManagedIdentityExtension"}}" -$global:UseInstanceMetadata = "{{GetVariable "useInstanceMetadata"}}" +$global:UseManagedIdentityExtension="{{GetVariable "useManagedIdentityExtension"}}" +$global:UseInstanceMetadata="{{GetVariable "useInstanceMetadata"}}" -$global:LoadBalancerSku = "{{GetVariable "loadBalancerSku"}}" -$global:ExcludeMasterFromStandardLB = "{{GetVariable "excludeMasterFromStandardLB"}}" +$global:LoadBalancerSku="{{GetVariable "loadBalancerSku"}}" +$global:ExcludeMasterFromStandardLB="{{GetVariable "excludeMasterFromStandardLB"}}" -$global:PrivateEgressProxyAddress = "{{GetPrivateEgressProxyAddress}}" +$global:PrivateEgressProxyAddress="{{GetPrivateEgressProxyAddress}}" # Windows defaults, not changed by aks-engine -$global:CacheDir = "c:\akse-cache" -$global:KubeDir = "c:\k" -$global:HNSModule = [Io.path]::Combine("$global:KubeDir", "hns.v2.psm1") +$global:CacheDir="c:\akse-cache" +$global:KubeDir="c:\k" +$global:HNSModule=[Io.path]::Combine("$global:KubeDir", "hns.v2.psm1") -$global:KubeDnsSearchPath = "svc.cluster.local" +$global:KubeDnsSearchPath="svc.cluster.local" -$global:CNIPath = [Io.path]::Combine("$global:KubeDir", "cni") -$global:NetworkMode = "L2Bridge" -$global:CNIConfig = [Io.path]::Combine($global:CNIPath, "config", "`$global:NetworkMode.conf") -$global:CNIConfigPath = [Io.path]::Combine("$global:CNIPath", "config") +$global:CNIPath=[Io.path]::Combine("$global:KubeDir", "cni") +$global:NetworkMode="L2Bridge" +$global:CNIConfig=[Io.path]::Combine($global:CNIPath, "config", "`$global:NetworkMode.conf") +$global:CNIConfigPath=[Io.path]::Combine("$global:CNIPath", "config") -$global:AzureCNIDir = [Io.path]::Combine("$global:KubeDir", "azurecni") -$global:AzureCNIBinDir = [Io.path]::Combine("$global:AzureCNIDir", "bin") -$global:AzureCNIConfDir = [Io.path]::Combine("$global:AzureCNIDir", "netconf") +$global:AzureCNIDir=[Io.path]::Combine("$global:KubeDir", "azurecni") +$global:AzureCNIBinDir=[Io.path]::Combine("$global:AzureCNIDir", "bin") +$global:AzureCNIConfDir=[Io.path]::Combine("$global:AzureCNIDir", "netconf") # Azure cni configuration -# $global:NetworkPolicy = "{{GetParameter "networkPolicy"}}" # BUG: unused -$global:NetworkPlugin = "{{GetParameter "networkPlugin"}}" -$global:VNetCNIPluginsURL = "{{GetParameter "vnetCniWindowsPluginsURL"}}" -$global:IsDualStackEnabled = {{if IsIPv6DualStackFeatureEnabled}}$true{{else}}$false{{end}} -$global:IsAzureCNIOverlayEnabled = {{if IsAzureCNIOverlayFeatureEnabled}}$true{{else}}$false{{end}} -$global:CiliumDataplaneEnabled = {{if CiliumDataplaneEnabled}}$true{{else}}$false{{end}} -$global:IsIMDSRestrictionEnabled = {{if EnableIMDSRestriction}}$true{{else}}$false{{end}} +# $global:NetworkPolicy="{{GetParameter "networkPolicy"}}" # BUG: unused +$global:NetworkPlugin="{{GetParameter "networkPlugin"}}" +$global:VNetCNIPluginsURL="{{GetParameter "vnetCniWindowsPluginsURL"}}" +$global:IsDualStackEnabled={{if IsIPv6DualStackFeatureEnabled}}$true {{else}}$false {{end}} +$global:IsAzureCNIOverlayEnabled={{if IsAzureCNIOverlayFeatureEnabled}}$true {{else}}$false {{end}} +$global:CiliumDataplaneEnabled={{if CiliumDataplaneEnabled}}$true {{else}}$false {{end}} +$global:IsIMDSRestrictionEnabled={{if EnableIMDSRestriction}}$true {{else}}$false {{end}} # Kubelet credential provider -$global:CredentialProviderURL = "{{GetParameter "windowsCredentialProviderURL"}}" +$global:CredentialProviderURL="{{GetParameter "windowsCredentialProviderURL"}}" # CSI Proxy settings -$global:EnableCsiProxy = [System.Convert]::ToBoolean("{{GetVariable "windowsEnableCSIProxy" }}"); -$global:CsiProxyUrl = "{{GetVariable "windowsCSIProxyURL" }}"; +$global:EnableCsiProxy=[System.Convert]::ToBoolean("{{GetVariable "windowsEnableCSIProxy"}}"); +$global:CsiProxyUrl="{{GetVariable "windowsCSIProxyURL"}}"; # Hosts Config Agent settings -$global:EnableHostsConfigAgent = [System.Convert]::ToBoolean("{{ EnableHostsConfigAgent }}"); +$global:EnableHostsConfigAgent=[System.Convert]::ToBoolean("{{EnableHostsConfigAgent}}"); # These scripts are used by cse -$global:CSEScriptsPackageUrl = "{{GetVariable "windowsCSEScriptsPackageURL" }}"; +$global:CSEScriptsPackageUrl="{{GetVariable "windowsCSEScriptsPackageURL"}}"; # The windows nvidia gpu driver related url is used by windows cse -$global:GpuDriverURL = "{{GetVariable "windowsGpuDriverURL" }}"; +$global:GpuDriverURL="{{GetVariable "windowsGpuDriverURL"}}"; # PauseImage -$global:WindowsPauseImageURL = "{{GetVariable "windowsPauseImageURL" }}"; -$global:AlwaysPullWindowsPauseImage = [System.Convert]::ToBoolean("{{GetVariable "alwaysPullWindowsPauseImage" }}"); +$global:WindowsPauseImageURL="{{GetVariable "windowsPauseImageURL"}}"; +$global:AlwaysPullWindowsPauseImage=[System.Convert]::ToBoolean("{{GetVariable "alwaysPullWindowsPauseImage"}}"); # Calico -$global:WindowsCalicoPackageURL = "{{GetVariable "windowsCalicoPackageURL" }}"; +$global:WindowsCalicoPackageURL="{{GetVariable "windowsCalicoPackageURL"}}"; ## GPU install -$global:ConfigGPUDriverIfNeeded = [System.Convert]::ToBoolean("{{GetVariable "configGPUDriverIfNeeded" }}"); +$global:ConfigGPUDriverIfNeeded=[System.Convert]::ToBoolean("{{GetVariable "configGPUDriverIfNeeded"}}"); # GMSA -$global:WindowsGmsaPackageUrl = "{{GetVariable "windowsGmsaPackageUrl" }}"; +$global:WindowsGmsaPackageUrl="{{GetVariable "windowsGmsaPackageUrl"}}"; # TLS Bootstrap Token -$global:TLSBootstrapToken = "{{GetTLSBootstrapTokenForKubeConfig}}" +$global:TLSBootstrapToken="{{GetTLSBootstrapTokenForKubeConfig}}" # Secure TLS Bootstrap settings -$global:EnableSecureTLSBootstrapping = [System.Convert]::ToBoolean("{{EnableSecureTLSBootstrapping}}"); -$global:SecureTLSBootstrappingAADResource = "{{GetSecureTLSBootstrappingAADResource}}"; -$global:SecureTLSBootstrappingUserAssignedIdentityID = "{{GetSecureTLSBootstrappingUserAssignedIdentityID}}"; -$global:CustomSecureTLSBootstrappingClientDownloadURL = "{{GetCustomSecureTLSBootstrappingClientDownloadURL}}"; -$global:SecureTLSBootstrappingValidateKubeconfigTimeout = "{{GetSecureTLSBootstrappingValidateKubeconfigTimeout}}"; -$global:SecureTLSBootstrappingGetAccessTokenTimeout = "{{GetSecureTLSBootstrappingGetAccessTokenTimeout}}"; -$global:SecureTLSBootstrappingGetInstanceDataTimeout = "{{GetSecureTLSBootstrappingGetInstanceDataTimeout}}"; -$global:SecureTLSBootstrappingGetNonceTimeout = "{{GetSecureTLSBootstrappingGetNonceTimeout}}"; -$global:SecureTLSBootstrappingGetAttestedDataTimeout = "{{GetSecureTLSBootstrappingGetAttestedDataTimeout}}"; -$global:SecureTLSBootstrappingGetCredentialTimeout = "{{GetSecureTLSBootstrappingGetCredentialTimeout}}"; +$global:EnableSecureTLSBootstrapping=[System.Convert]::ToBoolean("{{EnableSecureTLSBootstrapping}}"); +$global:SecureTLSBootstrappingAADResource="{{GetSecureTLSBootstrappingAADResource}}"; +$global:SecureTLSBootstrappingUserAssignedIdentityID="{{GetSecureTLSBootstrappingUserAssignedIdentityID}}"; +$global:CustomSecureTLSBootstrappingClientDownloadURL="{{GetCustomSecureTLSBootstrappingClientDownloadURL}}"; +$global:SecureTLSBootstrappingValidateKubeconfigTimeout="{{GetSecureTLSBootstrappingValidateKubeconfigTimeout}}"; +$global:SecureTLSBootstrappingGetAccessTokenTimeout="{{GetSecureTLSBootstrappingGetAccessTokenTimeout}}"; +$global:SecureTLSBootstrappingGetInstanceDataTimeout="{{GetSecureTLSBootstrappingGetInstanceDataTimeout}}"; +$global:SecureTLSBootstrappingGetNonceTimeout="{{GetSecureTLSBootstrappingGetNonceTimeout}}"; +$global:SecureTLSBootstrappingGetAttestedDataTimeout="{{GetSecureTLSBootstrappingGetAttestedDataTimeout}}"; +$global:SecureTLSBootstrappingGetCredentialTimeout="{{GetSecureTLSBootstrappingGetCredentialTimeout}}"; # uniquely identifies AKS's Entra ID application, see: https://learn.microsoft.com/en-us/azure/aks/kubelogin-authentication#how-to-use-kubelogin-with-aks # this is used by aks-secure-tls-bootstrap-client.exe when requesting AAD tokens # TODO(cameissner): remove once 2025-10B image is released -$global:AKSAADServerAppID = "6dae42f8-4368-4678-94ff-3960e28e3630" +$global:AKSAADServerAppID="6dae42f8-4368-4678-94ff-3960e28e3630" # Disable OutBoundNAT in Azure CNI configuration -$global:IsDisableWindowsOutboundNat = [System.Convert]::ToBoolean("{{GetVariable "isDisableWindowsOutboundNat" }}"); +$global:IsDisableWindowsOutboundNat=[System.Convert]::ToBoolean("{{GetVariable "isDisableWindowsOutboundNat"}}"); # Base64 representation of ZIP archive -$zippedFiles = "{{ GetKubernetesWindowsAgentFunctions }}" +$zippedFiles="{{GetKubernetesWindowsAgentFunctions}}" -$global:KubeClusterConfigPath = "c:\k\kubeclusterconfig.json" -$fipsEnabled = [System.Convert]::ToBoolean("{{ FIPSEnabled }}") +$global:KubeClusterConfigPath="c:\k\kubeclusterconfig.json" +$fipsEnabled=[System.Convert]::ToBoolean("{{FIPSEnabled}}") # HNS remediator -$global:HNSRemediatorIntervalInMinutes = [System.Convert]::ToUInt32("{{GetHnsRemediatorIntervalInMinutes}}"); +$global:HNSRemediatorIntervalInMinutes=[System.Convert]::ToUInt32("{{GetHnsRemediatorIntervalInMinutes}}"); # Log generator -$global:LogGeneratorIntervalInMinutes = [System.Convert]::ToUInt32("{{GetLogGeneratorIntervalInMinutes}}"); +$global:LogGeneratorIntervalInMinutes=[System.Convert]::ToUInt32("{{GetLogGeneratorIntervalInMinutes}}"); -$global:EnableIncreaseDynamicPortRange = $false +$global:EnableIncreaseDynamicPortRange=$false -$global:RebootNeeded = $false +$global:RebootNeeded=$false -$global:IsSkipCleanupNetwork = [System.Convert]::ToBoolean("{{GetVariable "isSkipCleanupNetwork" }}"); -$PreProvisionOnly = [System.Convert]::ToBoolean("{{GetPreProvisionOnly}}"); +$global:IsSkipCleanupNetwork=[System.Convert]::ToBoolean("{{GetVariable "isSkipCleanupNetwork"}}"); +$PreProvisionOnly=[System.Convert]::ToBoolean("{{GetPreProvisionOnly}}"); -$global:EnableKubeletServingCertificateRotation = [System.Convert]::ToBoolean("{{EnableKubeletServingCertificateRotation}}") +$global:EnableKubeletServingCertificateRotation=[System.Convert]::ToBoolean("{{EnableKubeletServingCertificateRotation}}") # Windows Cilium Networking (WCN) Platform configuration -$global:EnableWindowsCiliumNetworking = [System.Convert]::ToBoolean("{{GetVariable "nextGenNetworkingEnabled" }}"); -$global:WindowsCiliumNetworkingConfiguration = "{{GetVariable "nextGenNetworkingConfig" }}"; -$global:WindowsCiliumNetworkingPath = Join-Path -Path $global:cacheDir -ChildPath 'wcn' -$global:WindowsCiliumInstallPath = Join-Path -Path $global:WindowsCiliumNetworkingPath -ChildPath 'install' +$global:EnableWindowsCiliumNetworking=[System.Convert]::ToBoolean("{{GetVariable "nextGenNetworkingEnabled"}}"); +$global:WindowsCiliumNetworkingConfiguration="{{GetVariable "nextGenNetworkingConfig"}}"; +$global:WindowsCiliumNetworkingPath=Join-Path -Path $global:cacheDir -ChildPath 'wcn' +$global:WindowsCiliumInstallPath=Join-Path -Path $global:WindowsCiliumNetworkingPath -ChildPath 'install' # Network isolated cluster $global:BootstrapProfileContainerRegistryServer="{{GetBootstrapProfileContainerRegistryServer}}" $global:MCRRepositoryBase="{{GetMCRRepositoryBase}}" -$global:NetworkIsolatedClusterTestMode = [System.Convert]::ToBoolean("{{GetNetworkIsolatedClusterTestMode}}"); # for ab e2e only for local ab test with remote cse package +$global:NetworkIsolatedClusterTestMode=[System.Convert]::ToBoolean("{{GetNetworkIsolatedClusterTestMode}}"); # for ab e2e only for local ab test with remote cse package $global:OrasCacheDir="c:\aks-tools\oras\" # refer to components.json $global:OrasPath="c:\aks-tools\oras\oras.exe" $global:OrasOutput="c:\aks-tools\oras\oras_verbose.out" $global:OrasRegistryConfigFile="c:\aks-tools\oras\config.yaml" # oras registry auth config file, not used, but have to define to avoid error "Error: failed to get user home directory: $HOME is not defined" +$global:OperationId=New-Guid # Extract cse helper script from ZIP -[io.file]::WriteAllBytes("scripts.zip", [System.Convert]::FromBase64String($zippedFiles)) -try { - Expand-Archive scripts.zip -DestinationPath "C:\\AzureData\\" -Force -ErrorAction Stop -} catch { - $global:ErrorMessage = ("Failed to extract inline scripts.zip: $($_.Exception.Message)" -replace '\|', '%7C' ) - $global:ExitCode = 73 - exit 73 -} +function Get-HelperScripts { + [io.file]::WriteAllBytes("scripts.zip", [System.Convert]::FromBase64String($zippedFiles)) + try { + Expand-Archive scripts.zip -DestinationPath "C:\\AzureData\\" -Force -ErrorAction Stop + # Dot-source windowscsehelper.ps1 with functions that are called in this script + . c:\AzureData\windows\windowscsehelper.ps1 + # util functions only can be used after this line, for example, Write-Log + } + catch { + $global:ErrorMessage=("Failed to extract inline scripts.zip: $($_.Exception.Message)" -replace '\|', '%7C' ) + $global:ExitCode=73 + exit 73 + } -# Dot-source windowscsehelper.ps1 with functions that are called in this script -. c:\AzureData\windows\windowscsehelper.ps1 -# util functions only can be used after this line, for example, Write-Log - -$global:OperationId = New-Guid - -# Detect whether the operator has supplied a fully-qualified CSE scripts zip URL -# (as opposed to a base URL ending in "/"). A fully-qualified URL is a signal to -# *always* fetch and overwrite the cached scripts on the VHD, which is required -# for testing branch builds against baked VHDs that already contain a prior -# version of the CSE scripts. Production RP only ever sets a base URL ending in -# "/", so this branch is a no-op for production traffic. -$global:IsExplicitCSEScriptsPackageUrl = $false -if (-not [string]::IsNullOrWhiteSpace($global:CSEScriptsPackageUrl) -and -not $global:CSEScriptsPackageUrl.EndsWith("/")) { - $global:IsExplicitCSEScriptsPackageUrl = $true -} + # Detect whether the operator has supplied a fully-qualified CSE scripts zip URL + # (as opposed to a base URL ending in "/"). A fully-qualified URL is a signal to + # *always* fetch and overwrite the cached scripts on the VHD, which is required + # for testing branch builds against baked VHDs that already contain a prior + # version of the CSE scripts. Production RP only ever sets a base URL ending in + # "/", so this branch is a no-op for production traffic. + $global:IsExplicitCSEScriptsPackageUrl=$false + if (-not [string]::IsNullOrWhiteSpace($global:CSEScriptsPackageUrl) -and -not $global:CSEScriptsPackageUrl.EndsWith("/")) { + $global:IsExplicitCSEScriptsPackageUrl=$true + } -if ((-not (Test-Path "C:\AzureData\windows\azurecnifunc.ps1")) -or $global:IsExplicitCSEScriptsPackageUrl) { - # CSEScriptsPackage is cached on VHD. Previously the cse package version was managed in components.json, whereas RP set the package URL which is a storage account. - # From 2025-06 The CSE packages is released on the VHD. RP can use fully qualified URL to download CSE scripts package when required out of VHD release cycle. - # In the transition period, it is important that when deal with older VHD versions, the agentbaker runtime provision script needs to be compatible with the latest known storage account package, 0.0.52. - - $WindowsCSEScriptsPackage = "aks-windows-cse-scripts-current.zip" - $scriptsZip = $null - $shouldCleanup = $false - - # Step 1: Try to find cached scripts on VHD (skipped when an explicit - # CSEScriptsPackageUrl was supplied, so the operator-provided zip is the - # source of truth and always overwrites the cached copy). - if (-not $global:IsExplicitCSEScriptsPackageUrl -and $global:CacheDir -and (Test-Path $global:CacheDir)) { - $searchCachedScripts = [IO.Directory]::GetFiles($global:CacheDir, $WindowsCSEScriptsPackage, [IO.SearchOption]::AllDirectories) - Write-Log "the directory $global:CacheDir contains the following files:" - Get-ChildItem -Path $global:CacheDir | ForEach-Object { Write-Log " $_" } - if ($searchCachedScripts.Count -gt 0) { - $scriptsZip = $searchCachedScripts[0] - Write-Log "Found cached CSE scripts at $scriptsZip" + if ((-not (Test-Path "C:\AzureData\windows\azurecnifunc.ps1")) -or $global:IsExplicitCSEScriptsPackageUrl) { + # CSEScriptsPackage is cached on VHD. Previously the cse package version was managed in components.json, whereas RP set the package URL which is a storage account. + # From 2025-06 The CSE packages is released on the VHD. RP can use fully qualified URL to download CSE scripts package when required out of VHD release cycle. + # In the transition period, it is important that when deal with older VHD versions, the agentbaker runtime provision script needs to be compatible with the latest known storage account package, 0.0.52. + + $WindowsCSEScriptsPackage="aks-windows-cse-scripts-current.zip" + $scriptsZip=$null + $shouldCleanup=$false + + # Step 1: Try to find cached scripts on VHD (skipped when an explicit + # CSEScriptsPackageUrl was supplied, so the operator-provided zip is the + # source of truth and always overwrites the cached copy). + if (-not $global:IsExplicitCSEScriptsPackageUrl -and $global:CacheDir -and (Test-Path $global:CacheDir)) { + $searchCachedScripts=[IO.Directory]::GetFiles($global:CacheDir, $WindowsCSEScriptsPackage, [IO.SearchOption]::AllDirectories) + Write-Log "the directory $global:CacheDir contains the following files:" + Get-ChildItem -Path $global:CacheDir | ForEach-Object { Write-Log " $_" } + if ($searchCachedScripts.Count -gt 0) { + $scriptsZip=$searchCachedScripts[0] + Write-Log "Found cached CSE scripts at $scriptsZip" + } } - } - # Step 2: For non-network-isolated clusters, download scripts if needed (overrides cached version when appropriate) - $isNetworkIsolated = -not [string]::IsNullOrWhiteSpace($global:BootstrapProfileContainerRegistryServer) -and -not $global:NetworkIsolatedClusterTestMode - if (-not $isNetworkIsolated) { - Write-Log "Requested CSEScriptsPackageUrl is $global:CSEScriptsPackageUrl" - if ($global:CSEScriptsPackageUrl.EndsWith("/")) { + # Step 2: For non-network-isolated clusters, download scripts if needed (overrides cached version when appropriate) + $isNetworkIsolated=-not [string]::IsNullOrWhiteSpace($global:BootstrapProfileContainerRegistryServer) -and -not $global:NetworkIsolatedClusterTestMode + if (-not $isNetworkIsolated) { + Write-Log "Requested CSEScriptsPackageUrl is $global:CSEScriptsPackageUrl" + if ($global:CSEScriptsPackageUrl.EndsWith("/")) { + if (-not $scriptsZip) { + Write-Log "Could not find windows cse package on VHD. Use remote version instead." + $WindowsCSEScriptsPackage="aks-windows-cse-scripts-v0.0.52.zip" + } + Write-Log "WindowsCSEScriptsPackage is $WindowsCSEScriptsPackage" + $global:CSEScriptsPackageUrl=$global:CSEScriptsPackageUrl + $WindowsCSEScriptsPackage + } + Write-Log "CSEScriptsPackageUrl used for provision is $global:CSEScriptsPackageUrl" + + # Download CSE function scripts + $downloadedFile='c:\csescripts.zip' + Logs-To-Event -TaskName "AKS.WindowsCSE.DownloadAndExpandCSEScriptPackageUrl" -TaskMessage "Start to get CSE scripts. CSEScriptsPackageUrl: $global:CSEScriptsPackageUrl" + DownloadFileOverHttp -Url $global:CSEScriptsPackageUrl -DestinationPath $downloadedFile -ExitCode $global:WINDOWS_CSE_ERROR_DOWNLOAD_CSE_PACKAGE + $scriptsZip=$downloadedFile + $shouldCleanup=$true + } + else { + Write-Log "Network isolated cluster detected (BootstrapProfileContainerRegistryServer is set), skip CSE scripts download and use cached scripts" if (-not $scriptsZip) { - Write-Log "Could not find windows cse package on VHD. Use remote version instead." - $WindowsCSEScriptsPackage = "aks-windows-cse-scripts-v0.0.52.zip" + Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_NETWORK_ISOLATED_CLUSTER_CSE_NOT_CACHED -ErrorMessage "Cached CSE scripts package '$WindowsCSEScriptsPackage' not found under cache directory '$global:CacheDir'" } - Write-Log "WindowsCSEScriptsPackage is $WindowsCSEScriptsPackage" - $global:CSEScriptsPackageUrl = $global:CSEScriptsPackageUrl + $WindowsCSEScriptsPackage } - Write-Log "CSEScriptsPackageUrl used for provision is $global:CSEScriptsPackageUrl" - - # Download CSE function scripts - $downloadedFile = 'c:\csescripts.zip' - Logs-To-Event -TaskName "AKS.WindowsCSE.DownloadAndExpandCSEScriptPackageUrl" -TaskMessage "Start to get CSE scripts. CSEScriptsPackageUrl: $global:CSEScriptsPackageUrl" - DownloadFileOverHttp -Url $global:CSEScriptsPackageUrl -DestinationPath $downloadedFile -ExitCode $global:WINDOWS_CSE_ERROR_DOWNLOAD_CSE_PACKAGE - $scriptsZip = $downloadedFile - $shouldCleanup = $true - } else { - Write-Log "Network isolated cluster detected (BootstrapProfileContainerRegistryServer is set), skip CSE scripts download and use cached scripts" - if (-not $scriptsZip) { - Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_NETWORK_ISOLATED_CLUSTER_CSE_NOT_CACHED -ErrorMessage "Cached CSE scripts package '$WindowsCSEScriptsPackage' not found under cache directory '$global:CacheDir'" + + # Step 3: Extract scripts from the resolved zip + Write-Log "Extracting CSE scripts from $scriptsZip" + AKS-Expand-Archive -Path $scriptsZip -DestinationPath "C:\\AzureData\\windows" + if ($shouldCleanup) { + Remove-Item -Path $scriptsZip -Force } } + else { + Write-Log "CSE scripts already exist and no explicit CSEScriptsPackageUrl override, skipping download" + } - # Step 3: Extract scripts from the resolved zip - Write-Log "Extracting CSE scripts from $scriptsZip" - AKS-Expand-Archive -Path $scriptsZip -DestinationPath "C:\\AzureData\\windows" - if ($shouldCleanup) { - Remove-Item -Path $scriptsZip -Force + if (Test-Path -Path 'c:\AzureData\windows\all.ps1') { + . c:\AzureData\windows\all.ps1 + return } -} else { - Write-Log "CSE scripts already exist and no explicit CSEScriptsPackageUrl override, skipping download" -} -# Dot-source cse scripts with functions that are called in this script -. c:\AzureData\windows\azurecnifunc.ps1 -. c:\AzureData\windows\calicofunc.ps1 -. c:\AzureData\windows\configfunc.ps1 -. c:\AzureData\windows\containerdfunc.ps1 -. c:\AzureData\windows\kubeletfunc.ps1 -. c:\AzureData\windows\kubernetesfunc.ps1 -. c:\AzureData\windows\nvidiagpudriverfunc.ps1 - -if (Test-Path -Path 'c:\AzureData\windows\securetlsbootstrapfunc.ps1') { - . c:\AzureData\windows\securetlsbootstrapfunc.ps1 -} else { - Write-Log "Windows Secure TLS Bootstrap function script not found, skipping dot-source" -} + Write-Log "All CSE function script not found, falling back to individual import" -if (Test-Path -Path 'c:\AzureData\windows\windowsciliumnetworkingfunc.ps1') { - . c:\AzureData\windows\windowsciliumnetworkingfunc.ps1 -} else { - Write-Log "Windows Cilium Networking function script not found, skipping dot-source" -} + # Dot-source cse scripts with functions that are called in this script -if (Test-Path -Path 'c:\AzureData\windows\networkisolatedclusterfunc.ps1') { - . c:\AzureData\windows\networkisolatedclusterfunc.ps1 -} else { - Write-Log "Network Isolated Cluster function script not found, skipping dot-source" + # Dot-source cse scripts with functions that are called in this script + . c:\AzureData\windows\azurecnifunc.ps1 + . c:\AzureData\windows\calicofunc.ps1 + . c:\AzureData\windows\configfunc.ps1 + . c:\AzureData\windows\containerdfunc.ps1 + . c:\AzureData\windows\kubeletfunc.ps1 + . c:\AzureData\windows\kubernetesfunc.ps1 + . c:\AzureData\windows\nvidiagpudriverfunc.ps1 + + $optionalFunctionScripts = @( + 'c:\AzureData\windows\securetlsbootstrapfunc.ps1' + 'c:\AzureData\windows\windowsciliumnetworkingfunc.ps1' + 'c:\AzureData\windows\networkisolatedclusterfunc.ps1' + ) + + foreach ($script in $optionalFunctionScripts) { + if (Test-Path -Path $script) { + . $script + } else { + Write-Log "$script function script not found, skipping dot-source" + } + } } # ====== BASE PREP: BASE IMAGE PREPARATION ====== @@ -360,7 +369,7 @@ function BasePrep { # TODO update to use proxy # Install OpenSSH if SSH enabled - $sshEnabled = [System.Convert]::ToBoolean("{{ WindowsSSHEnabled }}") + $sshEnabled=[System.Convert]::ToBoolean("{{WindowsSSHEnabled}}") if ( $sshEnabled ) { Install-OpenSSH -SSHKeys $SSHKeys } @@ -377,9 +386,13 @@ function BasePrep { Create-Directory -FullPath "c:\k" Write-Log "Remove `"NT AUTHORITY\Authenticated Users`" write permissions on files in c:\k" icacls.exe "c:\k" /inheritance:r + if ($LASTEXITCODE -ne 0) { throw "icacls.exe failed to set inheritance on c:\k (exit code $LASTEXITCODE)" } icacls.exe "c:\k" /grant:r SYSTEM:`(OI`)`(CI`)`(F`) + if ($LASTEXITCODE -ne 0) { throw "icacls.exe failed to grant SYSTEM permissions on c:\k (exit code $LASTEXITCODE)" } icacls.exe "c:\k" /grant:r BUILTIN\Administrators:`(OI`)`(CI`)`(F`) + if ($LASTEXITCODE -ne 0) { throw "icacls.exe failed to grant Administrators permissions on c:\k (exit code $LASTEXITCODE)" } icacls.exe "c:\k" /grant:r BUILTIN\Users:`(OI`)`(CI`)`(RX`) + if ($LASTEXITCODE -ne 0) { throw "icacls.exe failed to grant Users permissions on c:\k (exit code $LASTEXITCODE)" } Write-Log "c:\k permissions: " icacls.exe "c:\k" Get-ProvisioningScripts @@ -395,12 +408,13 @@ function BasePrep { # oras initialization, including install and login, must be in front of Install-CredentialProvider, Get-KubePackage and Install-Containerd-Based-On-Kubernetes-Version if ((Test-Path variable:global:BootstrapProfileContainerRegistryServer) -and - -not [string]::IsNullOrWhiteSpace($global:BootstrapProfileContainerRegistryServer)) { - # variable exists and is not empty/whitespace + -not [string]::IsNullOrWhiteSpace($global:BootstrapProfileContainerRegistryServer)) { + # variable exists and is not empty/whitespace if (Get-Command -Name Initialize-Oras -ErrorAction SilentlyContinue) { Logs-To-Event -TaskName "AKS.WindowsCSE.InitializeOras" -TaskMessage "Ensure oras is initialized for network isolated cluster" Initialize-Oras - } else { + } + else { Write-Log "Initialize-Oras is not a recognized function, will skip oras initialization for network isolated cluster" } } @@ -408,19 +422,20 @@ function BasePrep { # to ensure we don't introduce any incompatibility between base CSE + CSE package versions if (Get-Command -Name Install-SecureTLSBootstrapClient -ErrorAction SilentlyContinue) { Install-SecureTLSBootstrapClient -KubeDir $global:KubeDir -CustomSecureTLSBootstrapClientDownloadUrl $global:CustomSecureTLSBootstrappingClientDownloadURL - } else { + } + else { Write-Log "Install-SecureTLSBootstrapClient is not a recognized function, will skip installation of the secure TLS bootstrap client" } - Install-CredentialProvider -KubeDir $global:KubeDir -CustomCloudContainerRegistryDNSSuffix {{if IsAKSCustomCloud}}"{{ AKSCustomCloudContainerRegistryDNSSuffix }}"{{else}}""{{end}} + Install-CredentialProvider -KubeDir $global:KubeDir -CustomCloudContainerRegistryDNSSuffix {{if IsAKSCustomCloud}}"{{AKSCustomCloudContainerRegistryDNSSuffix}}" {{else}}"" {{end}} Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL - $cniBinPath = $global:AzureCNIBinDir - $cniConfigPath = $global:AzureCNIConfDir + $cniBinPath=$global:AzureCNIBinDir + $cniConfigPath=$global:AzureCNIConfDir if ($global:NetworkPlugin -eq "kubenet") { - $cniBinPath = $global:CNIPath - $cniConfigPath = $global:CNIConfigPath + $cniBinPath=$global:CNIPath + $cniConfigPath=$global:CNIConfigPath } Install-Containerd-Based-On-Kubernetes-Version -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath -KubeDir $global:KubeDir -KubernetesVersion $global:KubeBinariesVersion @@ -444,7 +459,7 @@ function BasePrep { Update-DefenderPreferences - $windowsVersion = Get-WindowsVersion + $windowsVersion=Get-WindowsVersion if ($windowsVersion -eq "1809" -or $windowsVersion -eq "ltsc2022") { Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Start to enable secure TLS protocols" try { @@ -530,11 +545,13 @@ function NodePrep { -UseInstanceMetadata $global:UseInstanceMetadata ` -LoadBalancerSku $global:LoadBalancerSku ` -ExcludeMasterFromStandardLB $global:ExcludeMasterFromStandardLB ` - -TargetEnvironment {{if IsAKSCustomCloud}}"AzureStackCloud"{{else}}$TargetEnvironment{{end}} + -TargetEnvironment {{if IsAKSCustomCloud}}"AzureStackCloud" {{else}}$TargetEnvironment {{end}} + # we borrow the logic of AzureStackCloud to achieve AKSCustomCloud. + # In case of AKSCustomCloud, customer cloud env will be loaded from azurestackcloud.json {{if IsAKSCustomCloud}} - $azureStackConfigFile = [io.path]::Combine($global:KubeDir, "azurestackcloud.json") - $envJSON = "{{ GetBase64EncodedEnvironmentJSON }}" + $azureStackConfigFile=[io.path]::Combine($global:KubeDir, "azurestackcloud.json") + $envJSON="{{GetBase64EncodedEnvironmentJSON}}" [io.file]::WriteAllBytes($azureStackConfigFile, [System.Convert]::FromBase64String($envJSON)) {{end}} @@ -543,13 +560,13 @@ function NodePrep { # newer versions accept -Location (optional) and -FailOnError. Bind only the # parameters the resolved function actually supports so a new VHD paired with # an older CSE zip does not fail with parameter-binding errors. - $getCACertsCmd = Get-Command -Name Get-CACertificates -ErrorAction Ignore - $getCACertsArgs = @{} + $getCACertsCmd=Get-Command -Name Get-CACertificates -ErrorAction Ignore + $getCACertsArgs=@{} if ($getCACertsCmd -and $getCACertsCmd.Parameters.ContainsKey('Location')) { - $getCACertsArgs['Location'] = $Location + $getCACertsArgs['Location']=$Location } if ($getCACertsCmd -and $getCACertsCmd.Parameters.ContainsKey('FailOnError')) { - $getCACertsArgs['FailOnError'] = $true + $getCACertsArgs['FailOnError']=$true } Get-CACertificates @getCACertsArgs @@ -592,22 +609,25 @@ function NodePrep { -AADClientSecret $([System.Text.Encoding]::ASCII.GetString([System.Convert]::FromBase64String($AADClientSecret))) ` -NetworkAPIVersion $NetworkAPIVersion ` -AzureEnvironmentFilePath $([io.path]::Combine($global:KubeDir, "azurestackcloud.json")) ` - -IdentitySystem "{{ GetIdentitySystem }}" + -IdentitySystem "{{GetIdentitySystem}}" } New-ExternalHnsNetwork -IsDualStackEnabled $global:IsDualStackEnabled # Turn off Firewall to enable pods to talk to service endpoints. (Kubelet should eventually do this) netsh advfirewall set allprofiles state off + if ($LASTEXITCODE -ne 0) { throw "netsh advfirewall failed to disable firewall (exit code $LASTEXITCODE)" } # To ensure we don't introduce any incompatibility between base CSE + CSE package versions if (Get-Command -Name Enable-WindowsCiliumNetworking -ErrorAction SilentlyContinue) { if ($global:EnableWindowsCiliumNetworking) { Enable-WindowsCiliumNetworking - } else { + } + else { Write-Log "Windows Cilium Networking is not enabled, will skip Windows Cilium Networking installation" } - } else { + } + else { Write-Log "Enable-WindowsCiliumNetworking is not a recognized function, will skip Windows Cilium Networking installation" } @@ -616,7 +636,7 @@ function NodePrep { } if ($global:TLSBootstrapToken -or $global:EnableSecureTLSBootstrapping) { Write-Log "Removing temporary kube config" - $kubeConfigFile = [io.path]::Combine($KubeDir, "config") + $kubeConfigFile=[io.path]::Combine($KubeDir, "config") Remove-Item $kubeConfigFile } @@ -638,8 +658,7 @@ function NodePrep { Start-InstallGPUDriver -EnableInstall $global:ConfigGPUDriverIfNeeded -GpuDriverURL $global:GpuDriverURL - if (Test-Path $CacheDir) - { + if (Test-Path $CacheDir) { Write-Log "Removing aks cache directory" Remove-Item $CacheDir -Recurse -Force } @@ -649,16 +668,19 @@ function NodePrep { if ($global:RebootNeeded) { Logs-To-Event -TaskName "AKS.WindowsCSE.RestartComputer" -TaskMessage "Setup Complete, calling Postpone-RestartComputer with reboot" Postpone-RestartComputer - } else { + } + else { Logs-To-Event -TaskName "AKS.WindowsCSE.StartScheduledTask" -TaskMessage "Setup Complete, start NodeResetScriptTask to register Windows node without reboot" Start-NodeResetScriptTask } + Write-Log "NodePrep completed successfully" Logs-To-Event -TaskName "AKS.WindowsCSE.NodePrep" -TaskMessage "NodePrep completed successfully" } -try -{ +try { + . Get-HelperScripts + Logs-To-Event -TaskName "AKS.WindowsCSE.ExecuteCustomDataSetupScript" -TaskMessage ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AADClientId $AADClientId -NetworkAPIVersion $NetworkAPIVersion -TargetEnvironment $TargetEnvironment -CSEResultFilePath $CSEResultFilePath" # Exit early if the script has been executed @@ -688,24 +710,24 @@ try # when nodes are created from that VHD image. if (-not (Test-Path "C:\AzureData\base_prep.complete")) { BasePrep - } else { + } + else { Write-Log "Skipping basePrep - base_prep.complete file exists" } if (-not $PreProvisionOnly) { NodePrep - } else { + } + else { Write-Log "Skipping nodePrep - pre-provision only mode" } } -catch -{ +catch { Resolve-Error # Set-ExitCode will exit with the specified ExitCode immediately and not be caught by this catch block # Ideally all exceptions will be handled and no exception will be thrown. Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_UNKNOWN -ErrorMessage $_ } -finally -{ +finally { # Generate CSE result so it can be returned as the CSE response in csecmd.ps1 $ExecutionDuration=$(New-Timespan -Start $StartTime -End $(Get-Date)) Write-Log "CSE ExecutionDuration: $ExecutionDuration. ExitCode: $global:ExitCode" @@ -713,22 +735,25 @@ finally Logs-To-Event -TaskName "AKS.WindowsCSE.cse_main" -TaskMessage "ExitCode: $global:ExitCode. ErrorMessage: $global:ErrorMessage." # Create appropriate completion file based on mode - $completionFilePath = if ($PreProvisionOnly) { "C:\AzureData\base_prep.complete" } else { $CSEResultFilePath } + $completionFilePath=if ($PreProvisionOnly) { "C:\AzureData\base_prep.complete" } else { $CSEResultFilePath } if ($global:ExitCode -eq 0) { Set-Content -Path $completionFilePath -Value $global:ExitCode -Force - } else { - # $JsonString = "ExitCode: |{0}|, Output: |{1}|, Error: |{2}|" + } + else { + # $JsonString="ExitCode: |{0}|, Output: |{1}|, Error: |{2}|" # Max length of the full error message returned by Windows CSE is ~256. We use 240 to be safe. - $errorMessageLength = "ExitCode: |$global:ExitCode|, Output: |$($global:ErrorCodeNames[$global:ExitCode])|, Error: ||".Length - $turncatedErrorMessage = $global:ErrorMessage.Substring(0, [Math]::Min(240 - $errorMessageLength, $global:ErrorMessage.Length)) + $errorMessageLength="ExitCode: |$global:ExitCode|, Output: |$($global:ErrorCodeNames[$global:ExitCode])|, Error: ||".Length + $turncatedErrorMessage=$global:ErrorMessage.Substring(0, [Math]::Min(240 - $errorMessageLength, $global:ErrorMessage.Length)) Set-Content -Path $completionFilePath -Value "ExitCode: |$global:ExitCode|, Output: |$($global:ErrorCodeNames[$global:ExitCode])|, Error: |$turncatedErrorMessage|" } - if ($global:ExitCode -eq $global:WINDOWS_CSE_ERROR_DOWNLOAD_CSE_PACKAGE) { - Write-Log "Do not call Upload-GuestVMLogs because there is no cse script package downloaded" + # Upload-GuestVMLogs is defined in the CSE scripts package (configfunc.ps1). + # If the CSE scripts package failed to download or load, the function will not be available. + if (Get-Command -Name Upload-GuestVMLogs -ErrorAction SilentlyContinue) { + Upload-GuestVMLogs -ExitCode $global:ExitCode } else { - Upload-GuestVMLogs -ExitCode $global:ExitCode + Write-Log "Upload-GuestVMLogs is not available, skipping log upload (CSE scripts package may not have been loaded)" } } diff --git a/parts/windows/windowscsehelper.ps1 b/parts/windows/windowscsehelper.ps1 index 303d331c3ac..e7d77c12e6c 100644 --- a/parts/windows/windowscsehelper.ps1 +++ b/parts/windows/windowscsehelper.ps1 @@ -68,7 +68,7 @@ $global:WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_URL_NOT_EXE=61 $global:WINDOWS_CSE_ERROR_UPDATING_KUBE_CLUSTER_CONFIG=62 $global:WINDOWS_CSE_ERROR_GET_NODE_IPV6_IP=63 $global:WINDOWS_CSE_ERROR_GET_CONTAINERD_VERSION=64 -$global:WINDOWS_CSE_ERROR_INSTALL_CREDENTIAL_PROVIDER = 65 # exit code for installing credential provider +$global:WINDOWS_CSE_ERROR_INSTALL_CREDENTIAL_PROVIDER=65 # exit code for installing credential provider $global:WINDOWS_CSE_ERROR_DOWNLOAD_CREDEDNTIAL_PROVIDER=66 # exit code for downloading credential provider failure $global:WINDOWS_CSE_ERROR_CREDENTIAL_PROVIDER_CONFIG=67 # exit code for checking credential provider config failure $global:WINDOWS_CSE_ERROR_ADJUST_PAGEFILE_SIZE=68 @@ -93,7 +93,7 @@ $global:WINDOWS_CSE_ERROR_ORAS_PULL_CONTAINERD=84 # exit code for error pulling $global:WINDOWS_CSE_ERROR_MAX_CODE=85 # Please add new error code for downloading new packages in RP code too -$global:ErrorCodeNames = @( +$global:ErrorCodeNames=@( "WINDOWS_CSE_SUCCESS", "WINDOWS_CSE_ERROR_UNKNOWN", "WINDOWS_CSE_ERROR_DOWNLOAD_FILE_WITH_RETRY", @@ -182,33 +182,33 @@ $global:ErrorCodeNames = @( ) # The package domain to be used -$global:PackageDownloadFqdn = $null +$global:PackageDownloadFqdn=$null # The preferred package FQDN -$global:PreferredPackageDownloadFqdn = "packages.aks.azure.com" +$global:PreferredPackageDownloadFqdn="packages.aks.azure.com" # Fallback FQDN if preferred cannot be contacted -$global:FallbackPackageDownloadFqdn = "acs-mirror.azureedge.net" +$global:FallbackPackageDownloadFqdn="acs-mirror.azureedge.net" # NOTE: KubernetesVersion does not contain "v" -$global:MinimalKubernetesVersionWithLatestContainerd = "1.28.0" # Will change it to the correct version when we support new Windows containerd version +$global:MinimalKubernetesVersionWithLatestContainerd="1.28.0" # Will change it to the correct version when we support new Windows containerd version # The minimum kubernetes version to use containerd 2.x -$global:MinimalKubernetesVersionWithLatestContainerd2 = "1.33.0" +$global:MinimalKubernetesVersionWithLatestContainerd2="1.33.0" # Although the contianerd package url is set in AKS RP code now, we still need to update the following variables for AgentBaker Windows E2E tests. # Define containerd version template -$global:ContainerdPackageTemplate = "v{0}-azure.1/binaries/containerd-v{0}-azure.1-windows-amd64.tar.gz" +$global:ContainerdPackageTemplate="v{0}-azure.1/binaries/containerd-v{0}-azure.1-windows-amd64.tar.gz" # Version numbers only - used in various places -$global:StableContainerdVersion = "1.6.35" -$global:LatestContainerdVersion = "1.7.20" -$global:LatestContainerd2Version = "2.0.4" +$global:StableContainerdVersion="1.6.35" +$global:LatestContainerdVersion="1.7.20" +$global:LatestContainerd2Version="2.0.4" -$global:WindowsVersion2025 = "2025" +$global:WindowsVersion2025="2025" # Full package paths are generated using [string]::Format($global:ContainerdPackageTemplate, $version) when needed -$global:EventsLoggingDir = "C:\WindowsAzure\Logs\Plugins\Microsoft.Compute.CustomScriptExtension\Events\" -$global:TaskName = "" -$global:TaskTimeStamp = "" +$global:EventsLoggingDir="C:\WindowsAzure\Logs\Plugins\Microsoft.Compute.CustomScriptExtension\Events\" +$global:TaskName="" +$global:TaskTimeStamp="" # This filter removes null characters (\0) which are captured in nssm.exe output when logged through powershell filter RemoveNulls { $_ -replace '\0', '' } @@ -216,77 +216,75 @@ filter RemoveNulls { $_ -replace '\0', '' } filter Timestamp { "$(Get-Date -Format o): $_" } function Write-Log($message) { - $msg = $message | Timestamp - Write-Host $msg + $msg=$message | Timestamp + Write-Host $msg } function DownloadFileOverHttp { Param( - [Parameter(Mandatory = $true)][string] + [Parameter(Mandatory=$true)][string] $Url, - [Parameter(Mandatory = $true)][string] + [Parameter(Mandatory=$true)][string] $DestinationPath, - [Parameter(Mandatory = $true)][int] + [Parameter(Mandatory=$true)][int] $ExitCode ) # First check to see if a file with the same name is already cached on the VHD - $cleanUrl = $Url.Split('?')[0] - $fileName = [IO.Path]::GetFileName($cleanUrl) + $cleanUrl=$Url.Split('?')[0] + $fileName=[IO.Path]::GetFileName($cleanUrl) - $search = @() + $search=@() if ($global:CacheDir -and (Test-Path $global:CacheDir)) { - $search = [IO.Directory]::GetFiles($global:CacheDir, $fileName, [IO.SearchOption]::AllDirectories) + $search=[IO.Directory]::GetFiles($global:CacheDir, $fileName, [IO.SearchOption]::AllDirectories) } if ($search.Count -ne 0) { Write-Log "Using cached version of $fileName - Copying file from $($search[0]) to $DestinationPath" Copy-Item -Path $search[0] -Destination $DestinationPath -Force - } - else { - $secureProtocols = @() - $insecureProtocols = @([System.Net.SecurityProtocolType]::SystemDefault, [System.Net.SecurityProtocolType]::Ssl3) + } else { + $secureProtocols=@() + $insecureProtocols=@([System.Net.SecurityProtocolType]::SystemDefault, [System.Net.SecurityProtocolType]::Ssl3) foreach ($protocol in [System.Enum]::GetValues([System.Net.SecurityProtocolType])) { if ($insecureProtocols -notcontains $protocol) { $secureProtocols += $protocol } } - [System.Net.ServicePointManager]::SecurityProtocol = $secureProtocols + [System.Net.ServicePointManager]::SecurityProtocol=$secureProtocols - $MappedUrl = Update-BaseUrl -InitialUrl $Url + $MappedUrl=Update-BaseUrl -InitialUrl $Url Write-Log "Updated URL $Url -> $MappedUrl to download $fileName to $DestinationPath" - $oldProgressPreference = $ProgressPreference - $ProgressPreference = 'SilentlyContinue' + $oldProgressPreference=$ProgressPreference + $ProgressPreference='SilentlyContinue' - $downloadTimer = [System.Diagnostics.Stopwatch]::StartNew() + $downloadTimer=[System.Diagnostics.Stopwatch]::StartNew() try { - $args = @{Uri=$MappedUrl; Method="Get"; OutFile=$DestinationPath; ErrorAction="Stop"} - Retry-Command -Command "Invoke-RestMethod" -Args $args -Retries 5 -RetryDelaySeconds 10 + $arglist=@{Uri=$MappedUrl; Method="Get"; OutFile=$DestinationPath; ErrorAction="Stop" } + Retry-Command -Command "Invoke-RestMethod" -Args $arglist -Retries 5 -RetryDelaySeconds 10 } catch { Set-ExitCode -ExitCode $ExitCode -ErrorMessage "Failed in downloading $MappedUrl. Error: $_" } $downloadTimer.Stop() - $elapsedMs = $downloadTimer.ElapsedMilliseconds - - if ($global:AppInsightsClient -ne $null) { - $event = New-Object "Microsoft.ApplicationInsights.DataContracts.EventTelemetry" - $event.Name = "FileDownload" - $event.Properties["FileName"] = $fileName - $event.Metrics["DurationMs"] = $elapsedMs - $global:AppInsightsClient.TrackEvent($event) + $elapsedMs=$downloadTimer.ElapsedMilliseconds + + if ($null -ne $global:AppInsightsClient) { + $evt=New-Object "Microsoft.ApplicationInsights.DataContracts.EventTelemetry" + $evt.Name="FileDownload" + $evt.Properties["FileName"]=$fileName + $evt.Metrics["DurationMs"]=$elapsedMs + $global:AppInsightsClient.TrackEvent($evt) } - $ProgressPreference = $oldProgressPreference + $ProgressPreference=$oldProgressPreference Write-Log "Downloaded file $MappedUrl to $DestinationPath in $elapsedMs ms" Get-Item $DestinationPath -ErrorAction Continue | Format-List | Out-String | Write-Log } } -function Set-ExitCode -{ +function Set-ExitCode { Param( [Parameter(Mandatory=$true)][int] $ExitCode, @@ -300,24 +298,23 @@ function Set-ExitCode exit $ExitCode } -function Start-NodeResetScriptTask -{ +function Start-NodeResetScriptTask { Param( [Parameter(Mandatory=$false)][int] - $TimeoutSeconds = 180 + $TimeoutSeconds=180 ) - $taskName = "k8s-restart-job" - $taskRunningResult = 0x00041301 - $previousRunTime = (Get-ScheduledTaskInfo -TaskName $taskName).LastRunTime + $taskName="k8s-restart-job" + $taskRunningResult=0x00041301 + $previousRunTime=(Get-ScheduledTaskInfo -TaskName $taskName).LastRunTime Start-ScheduledTask -TaskName $taskName - $timer = [Diagnostics.Stopwatch]::StartNew() + $timer=[Diagnostics.Stopwatch]::StartNew() do { - $taskInfo = Get-ScheduledTaskInfo -TaskName $taskName - $task = Get-ScheduledTask -TaskName $taskName + $taskInfo=Get-ScheduledTaskInfo -TaskName $taskName + $task=Get-ScheduledTask -TaskName $taskName if ($task.State -eq "Ready" -and $taskInfo.LastRunTime -ne $previousRunTime) { - $taskInfo = Get-ScheduledTaskInfo -TaskName $taskName + $taskInfo=Get-ScheduledTaskInfo -TaskName $taskName if ($taskInfo.LastRunTime -ne $previousRunTime -and $taskInfo.LastTaskResult -ne $taskRunningResult) { break } @@ -336,7 +333,7 @@ function Start-NodeResetScriptTask Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_START_NODE_RESET_SCRIPT_TASK -ErrorMessage "NodeResetScriptTask failed with result $($taskInfo.LastTaskResult)" } - $kubeletService = Get-Service -Name "kubelet" -ErrorAction SilentlyContinue + $kubeletService=Get-Service -Name "kubelet" -ErrorAction SilentlyContinue if ($null -eq $kubeletService -or $kubeletService.Status -ne "Running") { Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_START_NODE_RESET_SCRIPT_TASK -ErrorMessage "kubelet service is not running after NodeResetScriptTask completed" } @@ -344,25 +341,23 @@ function Start-NodeResetScriptTask Write-Log -Message "We waited [$($timer.Elapsed.TotalSeconds)] seconds on NodeResetScriptTask" } -function Postpone-RestartComputer -{ +function Postpone-RestartComputer { Logs-To-Event -TaskName "AKS.WindowsCSE.PostponeRestartComputer" -TaskMessage "Start to create an one-time task to restart the VM" - $action = New-ScheduledTaskAction -Execute "powershell.exe" -Argument " -Command `"Restart-Computer -Force`"" - $principal = New-ScheduledTaskPrincipal -UserId SYSTEM -LogonType ServiceAccount -RunLevel Highest + $action=New-ScheduledTaskAction -Execute "powershell.exe" -Argument " -Command `"Restart-Computer -Force`"" + $principal=New-ScheduledTaskPrincipal -UserId SYSTEM -LogonType ServiceAccount -RunLevel Highest # trigger this task once - $trigger = New-JobTrigger -At (Get-Date).AddSeconds(15).DateTime -Once - $definition = New-ScheduledTask -Action $action -Principal $principal -Trigger $trigger -Description "Restart computer after provisioning the VM" + $trigger=New-JobTrigger -At (Get-Date).AddSeconds(15).DateTime -Once + $definition=New-ScheduledTask -Action $action -Principal $principal -Trigger $trigger -Description "Restart computer after provisioning the VM" Register-ScheduledTask -TaskName "restart-computer" -InputObject $definition Write-Log "Created an one-time task to restart the VM" } -function Create-Directory -{ +function Create-Directory { Param( [Parameter(Mandatory=$true)][string] $FullPath, [Parameter(Mandatory=$false)][string] - $DirectoryUsage = "general purpose" + $DirectoryUsage="general purpose" ) if (-Not (Test-Path $FullPath)) { @@ -375,19 +370,19 @@ function Create-Directory # https://stackoverflow.com/a/34559554/697126 function New-TemporaryDirectory { - $parent = [System.IO.Path]::GetTempPath() - [string] $name = [System.Guid]::NewGuid() + $parent=[System.IO.Path]::GetTempPath() + [string] $name=[System.Guid]::NewGuid() New-Item -ItemType Directory -Path (Join-Path $parent $name) } function AKS-Expand-Archive { Param( - [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][string]$Path, - [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][string]$DestinationPath, - [Parameter(Mandatory = $false)][ValidateNotNullOrEmpty()][boolean]$Force + [Parameter(Mandatory=$true)][ValidateNotNullOrEmpty()][string]$Path, + [Parameter(Mandatory=$true)][ValidateNotNullOrEmpty()][string]$DestinationPath, + [Parameter(Mandatory=$false)][ValidateNotNullOrEmpty()][boolean]$Force ) - try { + try { Expand-Archive -Path $Path -DestinationPath ${DestinationPath} -ErrorAction Stop -Force Write-Log "Successfully expanded file $Path to $DestinationPath" } catch { @@ -399,23 +394,22 @@ function AKS-Expand-Archive { function Retry-Command { Param( - [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][string] + [Parameter(Mandatory=$true)][ValidateNotNullOrEmpty()][string] $Command, - [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][hashtable] + [Parameter(Mandatory=$true)][ValidateNotNullOrEmpty()][hashtable] $Args, - [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][int] + [Parameter(Mandatory=$true)][ValidateNotNullOrEmpty()][int] $Retries, - [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][int] + [Parameter(Mandatory=$true)][ValidateNotNullOrEmpty()][int] $RetryDelaySeconds ) - for ($i = 0; ; ) { + for ($i=0; ; ) { try { # Do not log Args since Args may contain sensitive data Write-Log "Retry $i : $command" return & $Command @Args - } - catch { + } catch { $i++ if ($i -ge $Retries) { throw $_ @@ -434,22 +428,21 @@ function Invoke-Executable { [Parameter(Mandatory=$true)][int] $ExitCode, [int[]] - $AllowedExitCodes = @(0), + $AllowedExitCodes=@(0), [int] - $Retries = 0, + $Retries=0, [int] - $RetryDelaySeconds = 1 + $RetryDelaySeconds=1 ) - for ($i = 0; $i -le $Retries; $i++) { + for ($i=0; $i -le $Retries; $i++) { Write-Log "$i - Running $Executable $ArgList ..." & $Executable $ArgList if ($LASTEXITCODE -notin $AllowedExitCodes) { Write-Log "$Executable returned unsuccessfully with exit code $LASTEXITCODE" Start-Sleep -Seconds $RetryDelaySeconds continue - } - else { + } else { Write-Log "$Executable returned successfully" return } @@ -460,9 +453,9 @@ function Invoke-Executable { function Assert-FileExists { Param( - [Parameter(Mandatory = $true)][string] + [Parameter(Mandatory=$true)][string] $Filename, - [Parameter(Mandatory = $true)][int] + [Parameter(Mandatory=$true)][int] $ExitCode ) @@ -476,12 +469,12 @@ function Get-WindowsBuildNumber { } function Get-WindowsVersion { - $buildNumber = Get-WindowsBuildNumber + $buildNumber=Get-WindowsBuildNumber switch ($buildNumber) { "17763" { return "1809" } "20348" { return "ltsc2022" } "25398" { return "23H2" } - {$_ -ge "25399" -and $_ -le "30397"} { return $global:WindowsVersion2025 } + { $_ -ge "25399" -and $_ -le "30397" } { return $global:WindowsVersion2025 } Default { Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_NOT_FOUND_BUILD_NUMBER -ErrorMessage "Failed to find the windows build number: $buildNumber" } @@ -489,12 +482,12 @@ function Get-WindowsVersion { } function Get-WindowsPauseVersion { - $buildNumber = Get-WindowsBuildNumber + $buildNumber=Get-WindowsBuildNumber switch ($buildNumber) { "17763" { return "1809" } "20348" { return "ltsc2022" } "25398" { return "ltsc2022" } - {$_ -ge "25399" -and $_ -le "30397"} { return "ltsc2022" } + { $_ -ge "25399" -and $_ -le "30397" } { return "ltsc2022" } Default { Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_NOT_FOUND_BUILD_NUMBER -ErrorMessage "Failed to find the windows build number: $buildNumber" } @@ -502,65 +495,65 @@ function Get-WindowsPauseVersion { } function Install-Containerd-Based-On-Kubernetes-Version { - Param( - [Parameter(Mandatory = $true)][string] - $ContainerdUrl, - [Parameter(Mandatory = $true)][string] - $CNIBinDir, - [Parameter(Mandatory = $true)][string] - $CNIConfDir, - [Parameter(Mandatory = $true)][string] - $KubeDir, - [Parameter(Mandatory = $true)][string] - $KubernetesVersion - ) - - # Get the current Windows version, this is interim since we are progressively supporting containerd 2.0 for all Windows version. for now only test2025 - $windowsVersion = Get-WindowsVersion - Write-Log "Install Containerd with ContainerdURL: $ContainerdUrl, KubernetesVersion: $KubernetesVersion, WindowsVersion: $windowsVersion" - Logs-To-Event -TaskName "AKS.WindowsCSE.InstallContainerdBasedOnKubernetesVersion" -TaskMessage "Start to install ContainerD based on kubernetes version. ContainerdUrl: $global:ContainerdUrl, KubernetesVersion: $global:KubeBinariesVersion, Windows Version: $windowsVersion" - - # $global:ContainerdUrl is set from RP ContainerService.properties.orchestratorProfile.KubernetesConfig.WindowsContainerdURL - # it can be - # - a full URL. e.g., "https://packages.aks.azure.com/containerd/windows/v0.0.46/binaries/containerd-v0.0.46-windows-amd64.tar.gz" - # - an endpoint: e.g., "https://packages.aks.azure.com/containerd/windows/" - - # We only set containerd package based on kubernetes version when $global:ContainerdUrl ends with "/" so we support: - # 1. Current behavior to set the full URL - # 2. Setting containerd package in toggle for test purpose or hotfix - - $containerdVersion=$global:StableContainerdVersion - Write-Log "Install Containerd with request URL : $ContainerdUrl, Kubernetes version: $KubernetesVersion, Windows version: $windowsVersion." - - if ($ContainerdUrl.EndsWith("/")) { - # for now we only preview containerd 2.0 for Windows 2025 - if ($windowsVersion -eq $global:WindowsVersion2025) { - $containerdVersion=$global:LatestContainerd2Version - } elseif (([version]$KubernetesVersion).CompareTo([version]$global:MinimalKubernetesVersionWithLatestContainerd) -ge 0) { - $containerdVersion=$global:LatestContainerdVersion - } - $containerdPackage = [string]::Format($global:ContainerdPackageTemplate, $containerdVersion) - $ContainerdUrl = $ContainerdUrl + $containerdPackage - } elseif ( $windowsVersion -eq $global:WindowsVersion2025) { - # TODO (beileihuang) : remove this else if block when RP is release to set the correct versions for 2025 - $containerdPattern = "v\d+\.\d+\.\d+-azure\.\d+/binaries/containerd-v\d+\.\d+\.\d+-azure\.\d+-windows-amd64\.tar\.gz" - if ($ContainerdUrl -match $containerdPattern) { - $matchedPath = $matches[0] - $containerd2Package = [string]::Format($global:ContainerdPackageTemplate, $global:LatestContainerd2Version) - $ContainerdUrl = $ContainerdUrl.Replace($matchedPath, $containerd2Package) + Param( + [Parameter(Mandatory=$true)][string] + $ContainerdUrl, + [Parameter(Mandatory=$true)][string] + $CNIBinDir, + [Parameter(Mandatory=$true)][string] + $CNIConfDir, + [Parameter(Mandatory=$true)][string] + $KubeDir, + [Parameter(Mandatory=$true)][string] + $KubernetesVersion + ) + + # Get the current Windows version, this is interim since we are progressively supporting containerd 2.0 for all Windows version. for now only test2025 + $windowsVersion=Get-WindowsVersion + Write-Log "Install Containerd with ContainerdURL: $ContainerdUrl, KubernetesVersion: $KubernetesVersion, WindowsVersion: $windowsVersion" + Logs-To-Event -TaskName "AKS.WindowsCSE.InstallContainerdBasedOnKubernetesVersion" -TaskMessage "Start to install ContainerD based on kubernetes version. ContainerdUrl: $global:ContainerdUrl, KubernetesVersion: $global:KubeBinariesVersion, Windows Version: $windowsVersion" + + # $global:ContainerdUrl is set from RP ContainerService.properties.orchestratorProfile.KubernetesConfig.WindowsContainerdURL + # it can be + # - a full URL. e.g., "https://packages.aks.azure.com/containerd/windows/v0.0.46/binaries/containerd-v0.0.46-windows-amd64.tar.gz" + # - an endpoint: e.g., "https://packages.aks.azure.com/containerd/windows/" + + # We only set containerd package based on kubernetes version when $global:ContainerdUrl ends with "/" so we support: + # 1. Current behavior to set the full URL + # 2. Setting containerd package in toggle for test purpose or hotfix + + $containerdVersion=$global:StableContainerdVersion + Write-Log "Install Containerd with request URL : $ContainerdUrl, Kubernetes version: $KubernetesVersion, Windows version: $windowsVersion." + + if ($ContainerdUrl.EndsWith("/")) { + # for now we only preview containerd 2.0 for Windows 2025 + if ($windowsVersion -eq $global:WindowsVersion2025) { + $containerdVersion=$global:LatestContainerd2Version + } elseif (([version]$KubernetesVersion).CompareTo([version]$global:MinimalKubernetesVersionWithLatestContainerd) -ge 0) { + $containerdVersion=$global:LatestContainerdVersion + } + $containerdPackage=[string]::Format($global:ContainerdPackageTemplate, $containerdVersion) + $ContainerdUrl=$ContainerdUrl + $containerdPackage + } elseif ( $windowsVersion -eq $global:WindowsVersion2025) { + # TODO (beileihuang) : remove this else if block when RP is release to set the correct versions for 2025 + $containerdPattern="v\d+\.\d+\.\d+-azure\.\d+/binaries/containerd-v\d+\.\d+\.\d+-azure\.\d+-windows-amd64\.tar\.gz" + if ($ContainerdUrl -match $containerdPattern) { + $matchedPath=$matches[0] + $containerd2Package=[string]::Format($global:ContainerdPackageTemplate, $global:LatestContainerd2Version) + $ContainerdUrl=$ContainerdUrl.Replace($matchedPath, $containerd2Package) + } } - } - Write-Log "Install Containerd with resolved containerd pacakge url: $ContainerdUrl, Kubernetes version: $KubernetesVersion, Windows version: $windowsVersion." - Logs-To-Event -TaskName "AKS.WindowsCSE.InstallContainerd" -TaskMessage "Start to install ContainerD. ContainerdUrl: $ContainerdUrl" - Install-Containerd -ContainerdUrl $ContainerdUrl -CNIBinDir $CNIBinDir -CNIConfDir $CNIConfDir -KubeDir $KubeDir + Write-Log "Install Containerd with resolved containerd pacakge url: $ContainerdUrl, Kubernetes version: $KubernetesVersion, Windows version: $windowsVersion." + Logs-To-Event -TaskName "AKS.WindowsCSE.InstallContainerd" -TaskMessage "Start to install ContainerD. ContainerdUrl: $ContainerdUrl" + Install-Containerd -ContainerdUrl $ContainerdUrl -CNIBinDir $CNIBinDir -CNIConfDir $CNIConfDir -KubeDir $KubeDir } function Logs-To-Event { Param( - [Parameter(Mandatory = $true)][string] + [Parameter(Mandatory=$true)][string] $TaskName, - [Parameter(Mandatory = $true)][string] + [Parameter(Mandatory=$true)][string] $TaskMessage ) $eventLevel="Informational" @@ -571,19 +564,19 @@ function Logs-To-Event { $eventsFileName=[DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() $currentTime=$(Get-Date -Format "yyyy-MM-dd HH:mm:ss.fff") - $lastTaskName = "" - $lastTaskDuration = 0 + $lastTaskName="" + $lastTaskDuration=0 if ($global:TaskTimeStamp -ne "") { - $lastTaskName = $global:TaskName - $lastTaskDuration = $(New-Timespan -Start $global:TaskTimeStamp -End $currentTime) + $lastTaskName=$global:TaskName + $lastTaskDuration=$(New-Timespan -Start $global:TaskTimeStamp -End $currentTime) } - $global:TaskName = $TaskName - $global:TaskTimeStamp = $currentTime + $global:TaskName=$TaskName + $global:TaskTimeStamp=$currentTime Write-Log "$global:TaskName - $TaskMessage" - $TaskMessage = (echo $TaskMessage | ConvertTo-Json) - $messageJson = @" + $TaskMessage=(Write-Output $TaskMessage | ConvertTo-Json) + $messageJson=@" { "HostName": "$env:computername", "LastTaskName": "$lastTaskName", @@ -591,9 +584,9 @@ function Logs-To-Event { "CurrentTaskMessage": $TaskMessage } "@ - $messageJson = (echo $messageJson | ConvertTo-Json) + $messageJson=(Write-Output $messageJson | ConvertTo-Json) - $jsonString = @" + $jsonString=@" { "Timestamp": "$global:TaskTimeStamp", "OperationId": "$global:OperationId", @@ -614,46 +607,46 @@ function Logs-To-Event { # It will attempt to use the preferred FQDN first and if that fails it will fallback to the old CDN URL function Resolve-PackagesDownloadFqdn { Param( - [Parameter(Mandatory = $true)][string] + [Parameter(Mandatory=$true)][string] $PreferredFqdn, - [Parameter(Mandatory = $true)][string] + [Parameter(Mandatory=$true)][string] $FallbackFqdn, - [Parameter(Mandatory = $false)][int] - $Retries = 5, - [Parameter(Mandatory = $false)][int] - $WaitSleepSeconds = 1 + [Parameter(Mandatory=$false)][int] + $Retries=5, + [Parameter(Mandatory=$false)][int] + $WaitSleepSeconds=1 ) - $packageDownloadBaseUrl = $PreferredFqdn + $packageDownloadBaseUrl=$PreferredFqdn - for ($i = 1; $i -le $Retries; $i++) { + for ($i=1; $i -le $Retries; $i++) { # Confirm that we can establish connectivity to packages.aks.azure.com before node provisioning starts try { - $response = Invoke-WebRequest -Uri "https://${PreferredFqdn}/acs-mirror/healthz" -UseBasicParsing -TimeoutSec 5 -ErrorAction SilentlyContinue - $responseCode = [int]$response.StatusCode + $response=Invoke-WebRequest -Uri "https://${PreferredFqdn}/acs-mirror/healthz" -UseBasicParsing -TimeoutSec 5 -ErrorAction SilentlyContinue + $responseCode=[int]$response.StatusCode if ($responseCode -eq 200) { Write-Log "Established connectivity to $PreferredFqdn." | Out-Null break } } catch { - $responseCode = 0 + $responseCode=0 Write-Log "Exception while trying to establish connectivity to $PreferredFqdn. Exception: $_" | Out-Null if ($_.Exception.Response) { - $responseCode = [int]$_.Exception.Response.StatusCode + $responseCode=[int]$_.Exception.Response.StatusCode } } if ($i -eq $Retries) { # If we cannot establish connectivity to packages.aks.azure.com, fallback to old CDN URL - $packageDownloadBaseUrl = $FallbackFqdn + $packageDownloadBaseUrl=$FallbackFqdn break } else { Start-Sleep -Seconds $WaitSleepSeconds } } - $global:PackageDownloadFqdn = $packageDownloadBaseUrl + $global:PackageDownloadFqdn=$packageDownloadBaseUrl Logs-To-Event -TaskName "AKS.WindowsCSE.ResolvedPackageDomain" -TaskMessage "Package download FQDN: $global:PackageDownloadFqdn" } @@ -661,39 +654,38 @@ function Resolve-PackagesDownloadFqdn { # This function will swap the domain in the URL based on the verified package download FQDN function Update-BaseUrl { Param( - [Parameter(Mandatory = $true)][string] + [Parameter(Mandatory=$true)][string] $InitialUrl ) - $updatedUrl = $InitialUrl + $updatedUrl=$InitialUrl if (!($InitialUrl -match "acs-mirror\.azureedge\.net|packages\.aks\.azure\.com")) { # We're probably not in Public cloud return $updatedUrl } - if ($global:PackageDownloadFqdn -eq $null) { + if ($null -eq $global:PackageDownloadFqdn) { # We're in public cloud, but we haven't set the package download FQDN yet - $null = Resolve-PackagesDownloadFqdn -PreferredFqdn $global:PreferredPackageDownloadFqdn -FallbackFqdn $global:FallbackPackageDownloadFqdn + $null=Resolve-PackagesDownloadFqdn -PreferredFqdn $global:PreferredPackageDownloadFqdn -FallbackFqdn $global:FallbackPackageDownloadFqdn } # Replace domain based on the current package download FQDN if (($global:PackageDownloadFqdn -eq "packages.aks.azure.com") -and ($InitialUrl -like "https://acs-mirror.azureedge.net/*")) { - $updatedUrl = $InitialUrl -replace "acs-mirror.azureedge.net", $global:PackageDownloadFqdn + $updatedUrl=$InitialUrl -replace "acs-mirror.azureedge.net", $global:PackageDownloadFqdn } elseif (($global:PackageDownloadFqdn -eq "acs-mirror.azureedge.net") -and ($InitialUrl -like "https://packages.aks.azure.com/*")) { - $updatedUrl = $InitialUrl -replace "packages.aks.azure.com", $global:PackageDownloadFqdn + $updatedUrl=$InitialUrl -replace "packages.aks.azure.com", $global:PackageDownloadFqdn } return $updatedUrl } -function Resolve-Error ($ErrorRecord=$Error[0]) -{ - $ErrorRecord | Format-List * -Force - $ErrorRecord.InvocationInfo |Format-List * - $Exception = $ErrorRecord.Exception - for ($i = 0; $Exception; $i++, ($Exception = $Exception.InnerException)) - { "$i" * 80 - $Exception |Format-List * -Force - } +function Resolve-Error ($ErrorRecord=$Error[0]) { + $ErrorRecord | Format-List * -Force + $ErrorRecord.InvocationInfo | Format-List * + $Exception=$ErrorRecord.Exception + for ($i=0; $Exception; $i++, ($Exception=$Exception.InnerException)) { + "$i" * 80 + $Exception | Format-List * -Force + } } diff --git a/staging/cse/windows/all.ps1 b/staging/cse/windows/all.ps1 new file mode 100644 index 00000000000..4fc889eebc8 --- /dev/null +++ b/staging/cse/windows/all.ps1 @@ -0,0 +1,12 @@ + +# Dot-source cse scripts with functions that are bundled on the VHD +. c:\AzureData\windows\azurecnifunc.ps1 +. c:\AzureData\windows\calicofunc.ps1 +. c:\AzureData\windows\configfunc.ps1 +. c:\AzureData\windows\containerdfunc.ps1 +. c:\AzureData\windows\kubeletfunc.ps1 +. c:\AzureData\windows\kubernetesfunc.ps1 +. c:\AzureData\windows\nvidiagpudriverfunc.ps1 +. c:\AzureData\windows\securetlsbootstrapfunc.ps1 +. c:\AzureData\windows\windowsciliumnetworkingfunc.ps1 +. c:\AzureData\windows\networkisolatedclusterfunc.ps1 From 2686373413659306505ca584cd17cdc56efaa6d4 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 10 Aug 2026 16:55:12 +1000 Subject: [PATCH 19/38] chore(deps): update azure-cloud-node-manager (patch) (#9168) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- parts/common/components.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/parts/common/components.json b/parts/common/components.json index d5659e61442..b09d9b4e7d9 100644 --- a/parts/common/components.json +++ b/parts/common/components.json @@ -446,8 +446,8 @@ }, { "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes/azure-cloud-node-manager", - "latestVersion": "v1.36.4-1", - "previousLatestVersion": "v1.36.3-3" + "latestVersion": "v1.36.5-1", + "previousLatestVersion": "v1.36.4-1" } ], "windowsVersions": [ @@ -468,8 +468,8 @@ }, { "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes/azure-cloud-node-manager", - "latestVersion": "v1.36.4-windows-hpc-1", - "previousLatestVersion": "v1.36.3-windows-hpc-1" + "latestVersion": "v1.36.5-windows-hpc-1", + "previousLatestVersion": "v1.36.4-windows-hpc-1" } ] }, From ccbd214854301879fe2a949fa2f50aa30d9a8042 Mon Sep 17 00:00:00 2001 From: Martin Heberling Date: Mon, 10 Aug 2026 10:30:50 -0700 Subject: [PATCH 20/38] test(e2e): use the default VM SKU for kata and build the kata VHD in the PR gate (#9167) --- .pipelines/.vsts-vhd-builder.yaml | 26 ++++++++++++++++++++++++++ e2e/scenario_test.go | 10 ---------- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/.pipelines/.vsts-vhd-builder.yaml b/.pipelines/.vsts-vhd-builder.yaml index 99de84a985e..8e902d06e57 100644 --- a/.pipelines/.vsts-vhd-builder.yaml +++ b/.pipelines/.vsts-vhd-builder.yaml @@ -183,6 +183,32 @@ stages: parameters: artifactName: azurelinuxv3-gen2 + # Kata VHDs are built here so that VHD-side Kata changes (kata packages, erofs tooling, + # kata configuration files) are covered by the E2E stage below. Without this job no Kata + # image carries this build's buildId tag, so Test_AzureLinuxV3Gen2Kata silently skips + # (the E2E stage sets IgnoreScenariosWithMissingVhd: true). + - job: buildAzureLinuxV3gen2kata + timeoutInMinutes: 360 + steps: + - bash: | + echo '##vso[task.setvariable variable=OS_SKU]AzureLinux' + echo '##vso[task.setvariable variable=OS_VERSION]V3kata' + echo '##vso[task.setvariable variable=IMG_PUBLISHER]MicrosoftCBLMariner' + echo '##vso[task.setvariable variable=IMG_OFFER]azure-linux-3' + echo '##vso[task.setvariable variable=IMG_SKU]azure-linux-3-gen2' + echo '##vso[task.setvariable variable=IMG_VERSION]latest' + echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' + echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_D16ads_v5' + echo '##vso[task.setvariable variable=FEATURE_FLAGS]kata' + echo '##vso[task.setvariable variable=ARCHITECTURE]X86_64' + echo '##vso[task.setvariable variable=ENABLE_FIPS]false' + echo '##vso[task.setvariable variable=ENABLE_TRUSTED_LAUNCH]False' + echo '##vso[task.setvariable variable=ENABLE_CGROUPV2]True' + displayName: Setup Build Variables + - template: ./templates/.builder-release-template.yaml + parameters: + artifactName: azurelinuxv3-gen2-kata + - job: buildAzureLinuxV3ARM64gen2fips timeoutInMinutes: 360 steps: diff --git a/e2e/scenario_test.go b/e2e/scenario_test.go index 2792f170b40..aac3b224835 100644 --- a/e2e/scenario_test.go +++ b/e2e/scenario_test.go @@ -434,17 +434,10 @@ func Test_AzureLinuxV3Gen2Kata(t *testing.T) { Cluster: ClusterKubenet, VHD: config.VHDAzureLinuxV3Gen2Kata, BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { - // Kata launches each pod inside its own VM, which requires nested virtualization - // and more headroom than the 2-vCPU e2e default (config.Config.DefaultVMSKU). - nbc.ContainerService.Properties.AgentPoolProfiles[0].VMSize = kataVMSize - nbc.AgentPoolProfile.VMSize = kataVMSize // Leave unattended upgrades on so that CSE's kata-specific opt-out branch is // actually exercised, which ValidateKataHostReadiness asserts. nbc.DisableUnattendedUpgrades = false }, - VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { - vmss.SKU.Name = to.Ptr(kataVMSize) - }, Validator: func(ctx context.Context, s *Scenario) { ValidateKataContainerdConfig(ctx, s) ValidateKataContainerdConfigDump(ctx, s) @@ -457,9 +450,6 @@ func Test_AzureLinuxV3Gen2Kata(t *testing.T) { }) } -// kataVMSize is a nested-virtualization capable SKU with enough capacity to host a Kata guest VM. -const kataVMSize = "Standard_D4ds_v5" - func Test_AzureLinuxV3_CustomCA(t *testing.T) { RunScenario(t, &Scenario{ Description: "Tests that an AzureLinuxV3 node can be properly bootstrapped with custom ca", From ef0fecb665b4786fcf07c34fe7389d39135d76eb Mon Sep 17 00:00:00 2001 From: Xinhe Li Date: Tue, 11 Aug 2026 04:05:55 +1000 Subject: [PATCH 21/38] fix: add credential provider config for custom cloud network isolated clusters (#8962) Co-authored-by: Claude --- .../linux/cloud-init/artifacts/cse_config.sh | 44 +++++++------------ .../cloud-init/artifacts/cse_config_spec.sh | 33 +++++++++++++- 2 files changed, 46 insertions(+), 31 deletions(-) diff --git a/parts/linux/cloud-init/artifacts/cse_config.sh b/parts/linux/cloud-init/artifacts/cse_config.sh index 2c87414f18b..d0977e20026 100755 --- a/parts/linux/cloud-init/artifacts/cse_config.sh +++ b/parts/linux/cloud-init/artifacts/cse_config.sh @@ -1798,6 +1798,17 @@ writeCredentialProviderConfig() { - ${arg}" done fi + # matchImages and argument for network isolated cluster + local bootstrap_container_registry_match_image="" + local bootstrap_container_registry_args="" + if [ -n "${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}" ]; then + MCR_REPOSITORY_BASE="${MCR_REPOSITORY_BASE:=mcr.microsoft.com}" + MCR_REPOSITORY_BASE="${MCR_REPOSITORY_BASE%/}" + bootstrap_container_registry_match_image=" + - \"${MCR_REPOSITORY_BASE}\"" + bootstrap_container_registry_args=" + - --registry-mirror=${MCR_REPOSITORY_BASE}:${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}" + fi if [ -n "$AKS_CUSTOM_CLOUD_CONTAINER_REGISTRY_DNS_SUFFIX" ]; then echo "configure credential provider for custom cloud" @@ -1815,36 +1826,11 @@ providers: - "*.*.geo.azurecr.cn" - "*.*.geo.azurecr.de" - "*.*.geo.azurecr.us" - - "*$AKS_CUSTOM_CLOUD_CONTAINER_REGISTRY_DNS_SUFFIX" - defaultCacheDuration: "10m" - apiVersion: credentialprovider.kubelet.k8s.io/v1${ib_token_attributes} - args: - - /etc/kubernetes/azure.json${ib_args} -EOF - elif [ -n "${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}" ]; then - echo "configure credential provider for network isolated cluster" - MCR_REPOSITORY_BASE="${MCR_REPOSITORY_BASE:=mcr.microsoft.com}" - MCR_REPOSITORY_BASE="${MCR_REPOSITORY_BASE%/}" - tee "${config_file_path}" > /dev/null < Date: Tue, 11 Aug 2026 06:59:12 +0800 Subject: [PATCH 22/38] chore: upgrade Azure File CSI driver versions (#9163) --- parts/common/components.json | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/parts/common/components.json b/parts/common/components.json index b09d9b4e7d9..564dc60f7b4 100644 --- a/parts/common/components.json +++ b/parts/common/components.json @@ -532,8 +532,8 @@ "multiArchVersionsV2": [ { "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes-csi/azurefile-csi", - "latestVersion": "v1.35.6", - "previousLatestVersion": "v1.35.5" + "latestVersion": "v1.35.7", + "previousLatestVersion": "v1.35.6" }, { "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes-csi/azurefile-csi", @@ -542,15 +542,15 @@ }, { "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes-csi/azurefile-csi", - "latestVersion": "v1.34.7", - "previousLatestVersion": "v1.34.6" + "latestVersion": "v1.34.8", + "previousLatestVersion": "v1.34.7" } ], "windowsVersions": [ { "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes-csi/azurefile-csi", - "latestVersion": "v1.35.6-windows-hp", - "previousLatestVersion": "v1.35.5-windows-hp" + "latestVersion": "v1.35.7-windows-hp", + "previousLatestVersion": "v1.35.6-windows-hp" }, { "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes-csi/azurefile-csi", @@ -559,8 +559,8 @@ }, { "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes-csi/azurefile-csi", - "latestVersion": "v1.34.7-windows-hp", - "previousLatestVersion": "v1.34.6-windows-hp" + "latestVersion": "v1.34.8-windows-hp", + "previousLatestVersion": "v1.34.7-windows-hp" } ] }, From d7985360bfdaaabceb92c24b2273f4c5df0750f5 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 10 Aug 2026 16:01:00 -0700 Subject: [PATCH 23/38] chore(deps): update inspektor-gadget (patch) (#9139) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- parts/common/components.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/parts/common/components.json b/parts/common/components.json index 564dc60f7b4..efccf99a273 100644 --- a/parts/common/components.json +++ b/parts/common/components.json @@ -1770,7 +1770,7 @@ "versionsV2": [ { "renovateTag": "name=ig, repository=production, os=ubuntu, release=26.04", - "latestVersion": "0.54.1-ubuntu26.04u5" + "latestVersion": "0.54.1-ubuntu26.04u6" } ] }, @@ -1778,7 +1778,7 @@ "versionsV2": [ { "renovateTag": "name=ig, repository=production, os=ubuntu, release=24.04", - "latestVersion": "0.54.1-ubuntu24.04u5" + "latestVersion": "0.54.1-ubuntu24.04u6" } ] }, @@ -1786,7 +1786,7 @@ "versionsV2": [ { "renovateTag": "name=ig, repository=production, os=ubuntu, release=22.04", - "latestVersion": "0.54.1-ubuntu22.04u5" + "latestVersion": "0.54.1-ubuntu22.04u6" } ] }, @@ -1794,7 +1794,7 @@ "versionsV2": [ { "renovateTag": "name=ig, repository=production, os=ubuntu, release=20.04", - "latestVersion": "0.54.1-ubuntu20.04u5" + "latestVersion": "0.54.1-ubuntu20.04u6" } ] } @@ -1804,7 +1804,7 @@ "versionsV2": [ { "renovateTag": "RPM_registry=https://packages.microsoft.com/azurelinux/3.0/prod/cloud-native/x86_64/repodata, name=ig, os=azurelinux, release=3.0", - "latestVersion": "0.54.1-5.azl3" + "latestVersion": "0.54.1-6.azl3" } ] } From 57eaafca8ca0d2e48dd7ac2942dffcee85b0c1b6 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 10 Aug 2026 16:01:53 -0700 Subject: [PATCH 24/38] chore(deps): update kubernetes-cri-tools (patch) (#9140) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- parts/common/components.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/parts/common/components.json b/parts/common/components.json index efccf99a273..ef8e921b96b 100644 --- a/parts/common/components.json +++ b/parts/common/components.json @@ -1328,7 +1328,7 @@ "versionsV2": [ { "renovateTag": "name=kubernetes-cri-tools, repository=production, os=ubuntu, release=20.04", - "latestVersion": "1.34.0-ubuntu20.04u11" + "latestVersion": "1.34.0-ubuntu20.04u13" } ] }, @@ -1336,7 +1336,7 @@ "versionsV2": [ { "renovateTag": "name=kubernetes-cri-tools, repository=production, os=ubuntu, release=22.04", - "latestVersion": "1.34.0-ubuntu22.04u11" + "latestVersion": "1.34.0-ubuntu22.04u13" } ] }, @@ -1344,7 +1344,7 @@ "versionsV2": [ { "renovateTag": "name=kubernetes-cri-tools, repository=production, os=ubuntu, release=24.04", - "latestVersion": "1.34.0-ubuntu24.04u11" + "latestVersion": "1.34.0-ubuntu24.04u13" } ] }, @@ -1352,7 +1352,7 @@ "versionsV2": [ { "renovateTag": "name=kubernetes-cri-tools, repository=production, os=ubuntu, release=26.04", - "latestVersion": "1.34.0-ubuntu26.04u5" + "latestVersion": "1.34.0-ubuntu26.04u13" } ] } @@ -1382,7 +1382,7 @@ "versionsV2": [ { "renovateTag": "RPM_registry=https://packages.microsoft.com/azurelinux/3.0/prod/cloud-native/x86_64/repodata, name=kubernetes-cri-tools, os=azurelinux, release=3.0", - "latestVersion": "1.34.0-11.azl3" + "latestVersion": "1.34.0-12.azl3" } ] }, @@ -1400,7 +1400,7 @@ "versionsV2": [ { "renovateTag": "RPM_registry=https://packages.microsoft.com/azurelinux/3.0/prod/cloud-native/x86_64/repodata, name=kubernetes-cri-tools, os=azurelinux, release=3.0", - "latestVersion": "1.34.0-11.azl3" + "latestVersion": "1.34.0-12.azl3" } ] } From 248b17af74be268b83d8aa3d4340963225ef0714 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 10 Aug 2026 16:03:15 -0700 Subject: [PATCH 25/38] chore(deps): update dependency moby-containerd to v2.3.3-ubuntu26.04u2 (#9161) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- parts/common/components.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parts/common/components.json b/parts/common/components.json index ef8e921b96b..7c5e99b3c16 100644 --- a/parts/common/components.json +++ b/parts/common/components.json @@ -1043,7 +1043,7 @@ "versionsV2": [ { "renovateTag": "name=moby-containerd, repository=production, os=ubuntu, release=26.04", - "latestVersion": "2.3.2-ubuntu26.04u2" + "latestVersion": "2.3.3-ubuntu26.04u2" } ] }, From 1a16a14606f0802eba1f5e2b7157e46d0f24d80a Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 10 Aug 2026 16:03:50 -0700 Subject: [PATCH 26/38] chore(deps): update dependency node-exporter-kubernetes to v1.9.1-ubuntu26.04u23 (#9162) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- parts/common/components.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parts/common/components.json b/parts/common/components.json index 7c5e99b3c16..91428eb2bfb 100644 --- a/parts/common/components.json +++ b/parts/common/components.json @@ -2133,7 +2133,7 @@ "versionsV2": [ { "renovateTag": "name=node-exporter-kubernetes, repository=production, os=ubuntu, release=26.04", - "latestVersion": "1.9.1-ubuntu26.04u21" + "latestVersion": "1.9.1-ubuntu26.04u23" } ] }, From c6c17269975b171250f176ef53e234e300a477d7 Mon Sep 17 00:00:00 2001 From: chmill Date: Mon, 10 Aug 2026 23:16:47 +0000 Subject: [PATCH 27/38] fix: bound live patching kubeconfig wait --- .../artifacts/ubuntu/ubuntu-snapshot-update.sh | 11 ++++++++++- .../artifacts/ubuntu-snapshot-update_spec.sh | 13 ++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh b/parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh index e8545960baf..b2e986a2d41 100755 --- a/parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh +++ b/parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh @@ -15,13 +15,20 @@ set -e : "${KNEAD_COMPONENT_GOAL_ANNOTATION:=kubernetes.azure.com/live-patching-config-goal-hash}" : "${KNEAD_COMPONENT_STATUS_ANNOTATION:=kubernetes.azure.com/live-patching-status}" : "${KNEAD_COMPONENT_STATE_FILE:=/var/lib/aks/live-patching/current.json}" +: "${KNEAD_KUBECONFIG_WAIT_TIMEOUT_SECONDS:=600}" KNEAD_COMPONENT_RESULTS='{}' KNEAD_COMPONENT_RESULTS_VALID=true # Waits for kubelet credentials so kubectl can read Node and ConfigMap state. knead_wait_for_kubeconfig() { + local wait_started_at="${SECONDS}" + while [ ! -f "${KUBECONFIG}" ]; do + if [ $((SECONDS - wait_started_at)) -ge "${KNEAD_KUBECONFIG_WAIT_TIMEOUT_SECONDS}" ]; then + echo "timed out waiting for kubelet kubeconfig after ${KNEAD_KUBECONFIG_WAIT_TIMEOUT_SECONDS}s" >&2 + return 1 + fi echo "waiting for kubelet kubeconfig" sleep 3 done @@ -297,7 +304,9 @@ knead_main() { local payload local result=0 - knead_wait_for_kubeconfig + if ! knead_wait_for_kubeconfig; then + return 1 + fi if ! node_json="$(knead_read_node)"; then echo "failed to read node" return 1 diff --git a/spec/parts/linux/cloud-init/artifacts/ubuntu-snapshot-update_spec.sh b/spec/parts/linux/cloud-init/artifacts/ubuntu-snapshot-update_spec.sh index 0916782a840..78692c3f109 100644 --- a/spec/parts/linux/cloud-init/artifacts/ubuntu-snapshot-update_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/ubuntu-snapshot-update_spec.sh @@ -111,6 +111,17 @@ Describe 'ubuntu-snapshot-update.sh generic reconciliation' The output should not include 'annotate mock called' End + It 'fails after the kubeconfig wait deadline' + rm -f "${KUBECONFIG}" + KNEAD_KUBECONFIG_WAIT_TIMEOUT_SECONDS=0 + export KNEAD_KUBECONFIG_WAIT_TIMEOUT_SECONDS + + When call knead_main + The status should be failure + The error should include 'timed out waiting for kubelet kubeconfig after 0s' + The output should not include 'updateSecurityPatch called' + End + It 'does nothing when status is converged for the goal' TEST_GOAL="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" TEST_STATUS='{"currentHash":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","components":{"securityPatch":{"code":"Succeeded"}}}' @@ -346,4 +357,4 @@ Describe 'ubuntu-snapshot-update.sh generic reconciliation' The stderr should include 'parse error' The output should not include 'annotate mock called' End -End \ No newline at end of file +End From 8fc0fd7c97d403a14e509689b9a77462b2d4c94b Mon Sep 17 00:00:00 2001 From: Saewon Kwak <23280628+saewoni@users.noreply.github.com> Date: Tue, 18 Aug 2026 23:41:29 +0000 Subject: [PATCH 28/38] Add LocalDNS LPS bootstrap patching support Add AgentBaker LocalDNS live-patching support for the LPS bootstrap path and runtime knead dispatcher. The bootstrap path fetches LocalDNS nodeConfig from LPS, renders it through aks-node-controller, and feeds the generated Corefile into the existing updated.localdns.corefile flow before kubelet starts. The runtime path applies dispatched LocalDNS payloads with apply-localdns-config. Update focused unit, shellspec, and E2E coverage. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- aks-node-controller/app.go | 29 + aks-node-controller/go.mod | 16 +- aks-node-controller/go.sum | 51 +- aks-node-controller/localdnsconfig.go | 522 ++++++++++++++++++ aks-node-controller/localdnsconfig_test.go | 221 ++++++++ aks-node-controller/parser/helper.go | 6 +- aks-node-controller/parser/helper_test.go | 4 + .../parser/templates/localdns.toml.gtpl | 2 + e2e/scenario_localdns_hosts_test.go | 182 ++++++ parts/linux/cloud-init/artifacts/localdns.sh | 150 ++++- .../ubuntu/ubuntu-snapshot-update.sh | 55 ++ pkg/agent/baker.go | 2 + pkg/agent/baker_test.go | 10 + .../cloud-init/artifacts/localdns_spec.sh | 76 +++ .../artifacts/ubuntu-snapshot-update_spec.sh | 57 +- 15 files changed, 1375 insertions(+), 8 deletions(-) create mode 100644 aks-node-controller/localdnsconfig.go create mode 100644 aks-node-controller/localdnsconfig_test.go diff --git a/aks-node-controller/app.go b/aks-node-controller/app.go index cb0e3477779..b9453fcbb83 100644 --- a/aks-node-controller/app.go +++ b/aks-node-controller/app.go @@ -66,6 +66,8 @@ type App struct { // Authorization header for the check-hotfix LPS fetch. When nil, the real IMDS endpoint // is queried. fetchAttestedToken func(ctx context.Context) (string, error) + // fetchLocalDNSConfigFn overrides the real LPS LocalDNS config fetch for tests. + fetchLocalDNSConfigFn localDNSConfigFetcher } // provision.json values are emitted as strings by the shell jq invocation. @@ -168,6 +170,33 @@ func (a *App) Run(ctx context.Context, args []string) int { return a.runCheckHotfixCommand(ctx) }, }, + { + Name: "fetch-localdns-config", + Usage: "Read the LocalDNS config from the live-patching-service and update the Corefile (fail-open)", + Flags: []cli.Flag{ + &cli.StringFlag{Name: "output", Usage: "path to write the LocalDNS Corefile"}, + }, + Action: func(ctx context.Context, cmd *cli.Command) error { + if extra := cmd.Args().Slice(); len(extra) > 0 { + slog.Warn("ignoring unexpected fetch-localdns-config arguments", "args", strings.Join(extra, " ")) + } + return a.runFetchLocalDNSConfigCommand(ctx, cmd.String("output")) + }, + }, + { + Name: "apply-localdns-config", + Usage: "Apply a dispatched LocalDNS live-patching config slice to the Corefile", + Flags: []cli.Flag{ + &cli.StringFlag{Name: "config-file", Usage: "path to the LocalDNS config JSON; reads stdin when omitted or '-'"}, + &cli.StringFlag{Name: "output", Usage: "path to write the LocalDNS Corefile"}, + }, + Action: func(ctx context.Context, cmd *cli.Command) error { + if extra := cmd.Args().Slice(); len(extra) > 0 { + return fmt.Errorf("unexpected apply-localdns-config arguments: %s", strings.Join(extra, " ")) + } + return a.runApplyLocalDNSConfigCommand(ctx, cmd.String("config-file"), cmd.String("output"), cmd.Root().Writer) + }, + }, }, } diff --git a/aks-node-controller/go.mod b/aks-node-controller/go.mod index faf133532f5..a2f9e0c202f 100644 --- a/aks-node-controller/go.mod +++ b/aks-node-controller/go.mod @@ -3,18 +3,32 @@ module github.com/Azure/agentbaker/aks-node-controller go 1.25.11 require ( + github.com/Azure/agentbaker/aks-live-patching v0.0.0 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1 github.com/Masterminds/semver/v3 v3.5.0 github.com/fsnotify/fsnotify v1.8.0 github.com/google/go-cmp v0.7.0 github.com/stretchr/testify v1.11.1 github.com/urfave/cli/v3 v3.8.0 - google.golang.org/protobuf v1.36.7 + google.golang.org/grpc v1.82.1 + google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 ) require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/kr/pretty v0.3.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect + golang.org/x/net v0.55.0 // indirect golang.org/x/sys v0.45.0 // indirect + golang.org/x/text v0.37.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) + +replace github.com/Azure/agentbaker => ../ + +replace github.com/Azure/agentbaker/aks-live-patching => ../aks-live-patching + +replace github.com/coreos/ignition/v2 => github.com/flatcar/ignition/v2 v2.0.0-20250903113522-05b8a773288c diff --git a/aks-node-controller/go.sum b/aks-node-controller/go.sum index c58a6462885..4ab566d5771 100644 --- a/aks-node-controller/go.sum +++ b/aks-node-controller/go.sum @@ -2,23 +2,68 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1 h1:Wc1ml6QlJs2BHQ/9Bqu1jiyg github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM= github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE= github.com/Masterminds/semver/v3 v3.5.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/urfave/cli/v3 v3.8.0 h1:XqKPrm0q4P0q5JpoclYoCAv0/MIvH/jZ2umzuf8pNTI= github.com/urfave/cli/v3 v3.8.0/go.mod h1:ysVLtOEmg2tOy6PknnYVhDoouyC/6N42TMeoMzskhso= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A= -google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/aks-node-controller/localdnsconfig.go b/aks-node-controller/localdnsconfig.go new file mode 100644 index 00000000000..771d60b27c0 --- /dev/null +++ b/aks-node-controller/localdnsconfig.go @@ -0,0 +1,522 @@ +package main + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + akslivepatchingv1 "github.com/Azure/agentbaker/aks-live-patching/pkg/gen/akslivepatching/v1" + "github.com/Azure/agentbaker/aks-node-controller/helpers" + "github.com/Azure/agentbaker/aks-node-controller/parser" + aksnodeconfigv1 "github.com/Azure/agentbaker/aks-node-controller/pkg/gen/aksnodeconfig/v1" + "github.com/Azure/agentbaker/aks-node-controller/pkg/nodeconfigutils" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/encoding/protojson" +) + +const ( + localDNSLivePatchingComponentName = "localDNS" + defaultLocalDNSCorefilePath = "/opt/azure/containers/localdns/livepatched.localdns.corefile" + localDNSHostsFilePath = "/etc/localdns/hosts" + localDNSAgentPoolLabel = "kubernetes.azure.com/agentpool" + localDNSLPSALPNProto = "aks-live-patching" + localDNSALPNH2Proto = "h2" +) + +type localDNSConfigFetcher func(context.Context) (string, error) + +type localDNSConfigOutcome string + +const ( + outcomeLocalDNSConfigApplied localDNSConfigOutcome = "applied" + outcomeLocalDNSConfigAlreadyCurrent localDNSConfigOutcome = "alreadyCurrent" + outcomeLocalDNSConfigNotFound localDNSConfigOutcome = "notFound" + outcomeLocalDNSConfigNoCorefileData localDNSConfigOutcome = "noCorefileData" + outcomeLocalDNSConfigFailed localDNSConfigOutcome = "failed" +) + +type localDNSConfigPayload struct { + Corefile string `json:"corefile"` + CorefileBase64 string `json:"corefileBase64"` + CorefileBase64Alt string `json:"corefile_base64"` + CoreFile string `json:"coreFile"` + LocalDNSProfile json.RawMessage `json:"localDnsProfile"` + LocalDNSProfileAlt json.RawMessage `json:"local_dns_profile"` + AgentPools map[string]localDNSAgentPoolConfig `json:"agentPools"` + Profiles map[string]localDNSAgentPoolConfig `json:"profiles"` +} + +type localDNSAgentPoolConfig struct { + CorefileVersion string `json:"corefileVersion"` + ConfigChecksum string `json:"configChecksum"` + Corefile string `json:"corefile"` + CorefileBase64 string `json:"corefileBase64"` + CorefileBase64Alt string `json:"corefile_base64"` + CoreFile string `json:"coreFile"` + LocalDNSProfile json.RawMessage `json:"localDnsProfile"` + LocalDNSProfileAlt json.RawMessage `json:"local_dns_profile"` +} + +type localDNSCorefileUpdate struct { + corefile string + desiredVersion string + hasCorefile bool +} + +func (a *App) runApplyLocalDNSConfigCommand(ctx context.Context, configPath string, outputPath string, writer io.Writer) error { + config, err := readLocalDNSConfigInput(configPath) + if err != nil { + return err + } + outcome, err := a.applyLocalDNSConfig(ctx, config, outputPath) + if writer != nil { + _, _ = fmt.Fprintf(writer, "%s\n", outcome) + } + if err != nil { + return err + } + return nil +} + +func readLocalDNSConfigInput(configPath string) (string, error) { + if configPath == "" || configPath == "-" { + data, err := io.ReadAll(os.Stdin) + if err != nil { + return "", fmt.Errorf("reading localDNS config from stdin: %w", err) + } + return string(data), nil + } + data, err := os.ReadFile(configPath) + if err != nil { + return "", fmt.Errorf("reading localDNS config %s: %w", configPath, err) + } + return string(data), nil +} + +func (a *App) applyLocalDNSConfig(ctx context.Context, config string, outputPath string) (localDNSConfigOutcome, error) { + return a.fetchAndApplyLocalDNSConfigWithFetcher(ctx, outputPath, func(context.Context) (string, error) { + return config, nil + }) +} + +func (a *App) runFetchLocalDNSConfigCommand(ctx context.Context, outputPath string) (err error) { + slog.Info("aks-node-controller fetch-localdns-config started", "outputPath", outputPath) + startTime := time.Now() + defer func() { + if r := recover(); r != nil { + slog.Error("fetch-localdns-config panicked (fail-open)", "panic", r) + if a.eventLogger != nil { + a.eventLogger.LogEvent("FetchLocalDNSConfig", + fmt.Sprintf("fetch-localdns-config outcome=%s panic=%v", outcomeLocalDNSConfigFailed, r), + helpers.EventLevelError, startTime, time.Now()) + } + err = nil + } + }() + + outcome, err := a.fetchAndApplyLocalDNSConfig(ctx, outputPath) + level := helpers.EventLevelInformational + if outcome == outcomeLocalDNSConfigFailed { + level = helpers.EventLevelError + } + message := fmt.Sprintf("fetch-localdns-config outcome=%s", outcome) + if err != nil { + message = fmt.Sprintf("%s error=%s", message, err.Error()) + slog.Warn("fetch-localdns-config completed with error (fail-open)", "outcome", outcome, "error", err) + } else { + slog.Info("fetch-localdns-config completed", "outcome", outcome) + } + if a.eventLogger != nil { + a.eventLogger.LogEvent("FetchLocalDNSConfig", message, level, startTime, time.Now()) + } + return nil +} + +func (a *App) fetchAndApplyLocalDNSConfig(ctx context.Context, outputPath string) (localDNSConfigOutcome, error) { + return a.fetchAndApplyLocalDNSConfigWithFetcher(ctx, outputPath, a.fetchLocalDNSConfig) +} + +func (a *App) fetchAndApplyLocalDNSConfigWithFetcher(ctx context.Context, outputPath string, fetcher localDNSConfigFetcher) (localDNSConfigOutcome, error) { + if outputPath == "" { + outputPath = defaultLocalDNSCorefilePath + } + config, err := fetcher(ctx) + if err != nil { + if isLPSUnavailable(err) { + return outcomeLocalDNSConfigNotFound, nil + } + return outcomeLocalDNSConfigFailed, err + } + update, err := a.localDNSCorefileUpdateFromConfig(config) + if err != nil { + return outcomeLocalDNSConfigFailed, err + } + versionPath := localDNSCorefileVersionPath(outputPath) + if !update.hasCorefile { + return outcomeLocalDNSConfigNoCorefileData, nil + } + current, readErr := os.ReadFile(outputPath) + if readErr != nil && !os.IsNotExist(readErr) { + return outcomeLocalDNSConfigFailed, fmt.Errorf("reading localDNS corefile %s: %w", outputPath, readErr) + } + contentMatches := readErr == nil && bytes.Equal(current, []byte(update.corefile)) + if update.desiredVersion != "" { + currentVersion, err := readLocalDNSCorefileVersion(versionPath) + if err != nil { + return outcomeLocalDNSConfigFailed, err + } + if currentVersion == update.desiredVersion && contentMatches { + return outcomeLocalDNSConfigAlreadyCurrent, nil + } + } else if contentMatches { + return outcomeLocalDNSConfigAlreadyCurrent, nil + } + if err := writeLocalDNSCorefile(outputPath, update.corefile); err != nil { + return outcomeLocalDNSConfigFailed, err + } + if update.desiredVersion != "" { + if err := writeLocalDNSCorefileVersion(versionPath, update.desiredVersion); err != nil { + return outcomeLocalDNSConfigFailed, err + } + } + return outcomeLocalDNSConfigApplied, nil +} + +func (a *App) fetchLocalDNSConfig(ctx context.Context) (string, error) { + if a.fetchLocalDNSConfigFn != nil { + return a.fetchLocalDNSConfigFn(ctx) + } + return a.fetchLocalDNSConfigFromLPS(ctx) +} + +func (a *App) fetchLocalDNSConfigFromLPS(ctx context.Context) (string, error) { + fqdn, caPEM, err := a.lpsTargetFromNodeConfig() + if err != nil { + return "", fmt.Errorf("resolving LPS endpoint from node config: %w", err) + } + token, err := a.attestedToken(ctx) + if err != nil { + return "", fmt.Errorf("imds attested token: %w", err) + } + rootCAs, err := certPoolFromPEM(caPEM) + if err != nil { + return "", err + } + host := fqdn + if h, _, splitErr := net.SplitHostPort(fqdn); splitErr == nil { + host = h + } + target := net.JoinHostPort(host, lpsAPIServerPort) + tlsConfig := &tls.Config{ + MinVersion: tls.VersionTLS12, + RootCAs: rootCAs, + NextProtos: []string{localDNSLPSALPNProto, localDNSALPNH2Proto}, + InsecureSkipVerify: true, //nolint:gosec // SNI stays on the apiserver FQDN for ALPN routing; chain and hostname are verified below. + VerifyPeerCertificate: localDNSVerifyChainAgainstPool(rootCAs, lpsSNIHost), + } + ctx, cancel := context.WithTimeout(ctx, lpsFetchTimeout) + defer cancel() + + conn, err := grpc.NewClient( + target, + grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)), + ) + if err != nil { + return "", fmt.Errorf("creating LPS client: %w", err) + } + defer conn.Close() + + client := akslivepatchingv1.NewLivePatchingServiceClient(conn) + rpcCtx := metadata.AppendToOutgoingContext(ctx, "authorization", token) + resp, err := client.GetComponentConfig(rpcCtx, &akslivepatchingv1.GetComponentConfigRequest{ + ComponentName: localDNSLivePatchingComponentName, + }) + if err != nil { + if statusCode, ok := localDNSLPSUnavailableStatusCode(status.Code(err)); ok { + return "", &lpsUnavailableError{statusCode: statusCode} + } + return "", fmt.Errorf("get %s component config: %w", localDNSLivePatchingComponentName, err) + } + return resp.GetConfig(), nil +} + +func localDNSLPSUnavailableStatusCode(code codes.Code) (int, bool) { + switch code { + case codes.NotFound: + return http.StatusNotFound, true + case codes.PermissionDenied: + return http.StatusForbidden, true + case codes.Unauthenticated: + return http.StatusUnauthorized, true + default: + return 0, false + } +} + +func localDNSVerifyChainAgainstPool(pool *x509.CertPool, serverName string) func([][]byte, [][]*x509.Certificate) error { + return func(rawCerts [][]byte, _ [][]*x509.Certificate) error { + if len(rawCerts) == 0 { + return fmt.Errorf("server presented no certificates") + } + leaf, err := x509.ParseCertificate(rawCerts[0]) + if err != nil { + return fmt.Errorf("failed to parse server certificate: %w", err) + } + intermediates := x509.NewCertPool() + for _, raw := range rawCerts[1:] { + cert, err := x509.ParseCertificate(raw) + if err != nil { + return fmt.Errorf("failed to parse intermediate certificate: %w", err) + } + intermediates.AddCert(cert) + } + if _, err := leaf.Verify(x509.VerifyOptions{Roots: pool, Intermediates: intermediates, DNSName: serverName}); err != nil { + return fmt.Errorf("server certificate verification failed: %w", err) + } + return nil + } +} + +func certPoolFromPEM(caPEM []byte) (*x509.CertPool, error) { + if len(caPEM) == 0 { + return nil, fmt.Errorf("cluster CA unavailable from provision-config; refusing to fetch over unverified TLS") + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(caPEM) { + return nil, fmt.Errorf("failed to parse cluster CA PEM") + } + return pool, nil +} + +func (a *App) localDNSCorefileUpdateFromConfig(config string) (localDNSCorefileUpdate, error) { + config = strings.TrimSpace(config) + if config == "" { + return localDNSCorefileUpdate{}, nil + } + + var payload localDNSConfigPayload + if err := json.Unmarshal([]byte(config), &payload); err != nil { + return localDNSCorefileUpdate{}, fmt.Errorf("parsing localDNS LPS config: %w", err) + } + + selected, found, err := a.selectLocalDNSAgentPoolConfig(payload) + if err != nil || !found { + return localDNSCorefileUpdate{}, err + } + + update := localDNSCorefileUpdate{ + desiredVersion: firstNonEmpty(selected.CorefileVersion, selected.ConfigChecksum), + } + return a.localDNSCorefileUpdateFromAgentPoolConfig(selected, update) +} + +func (a *App) selectLocalDNSAgentPoolConfig(payload localDNSConfigPayload) (localDNSAgentPoolConfig, bool, error) { + selected := localDNSAgentPoolConfig{ + Corefile: payload.Corefile, + CorefileBase64: payload.CorefileBase64, + CorefileBase64Alt: payload.CorefileBase64Alt, + CoreFile: payload.CoreFile, + LocalDNSProfile: payload.LocalDNSProfile, + LocalDNSProfileAlt: payload.LocalDNSProfileAlt, + } + if len(payload.AgentPools) == 0 && len(payload.Profiles) == 0 { + return selected, true, nil + } + + agentPool, err := a.nodeAgentPoolName() + if err != nil { + return localDNSAgentPoolConfig{}, false, err + } + if selected, ok := payload.AgentPools[agentPool]; ok { + return selected, true, nil + } + if selected, ok := payload.Profiles[agentPool]; ok { + return selected, true, nil + } + return localDNSAgentPoolConfig{}, false, nil +} + +func (a *App) localDNSCorefileUpdateFromAgentPoolConfig(selected localDNSAgentPoolConfig, update localDNSCorefileUpdate) (localDNSCorefileUpdate, error) { + switch { + case strings.TrimSpace(selected.Corefile) != "": + update.corefile = selected.Corefile + update.hasCorefile = true + return update, nil + case strings.TrimSpace(selected.CoreFile) != "": + update.corefile = selected.CoreFile + update.hasCorefile = true + return update, nil + case strings.TrimSpace(selected.CorefileBase64) != "": + return update.withCorefileBase64(selected.CorefileBase64) + case strings.TrimSpace(selected.CorefileBase64Alt) != "": + return update.withCorefileBase64(selected.CorefileBase64Alt) + } + profileJSON := selected.LocalDNSProfile + if len(profileJSON) == 0 { + profileJSON = selected.LocalDNSProfileAlt + } + if len(profileJSON) == 0 { + if selected.CorefileVersion != "" || selected.ConfigChecksum != "" { + slog.Info("localDNS LPS config has only version/checksum; Corefile content is required for bootstrap mutation", + "corefileVersion", selected.CorefileVersion, "configChecksum", selected.ConfigChecksum) + } + return update, nil + } + profile := &aksnodeconfigv1.LocalDnsProfile{} + unmarshalOptions := protojson.UnmarshalOptions{DiscardUnknown: true} + if err := unmarshalOptions.Unmarshal(profileJSON, profile); err != nil { + return localDNSCorefileUpdate{}, fmt.Errorf("parsing localDNS profile: %w", err) + } + if !profile.GetEnableLocalDns() { + return update, nil + } + nodeConfig, err := a.nodeConfigWithLocalDNSProfile(profile) + if err != nil { + return localDNSCorefileUpdate{}, err + } + includeHostsPlugin := profile.GetEnableHostsPlugin() + if includeHostsPlugin { + if _, statErr := os.Stat(localDNSHostsFilePath); statErr != nil { + includeHostsPlugin = false + } + } + corefile, err := parser.GenerateLocalDNSCorefileFromAKSNodeConfig(nodeConfig, includeHostsPlugin) + if err != nil { + return localDNSCorefileUpdate{}, err + } + update.corefile = corefile + update.hasCorefile = true + return update, nil +} + +func (u localDNSCorefileUpdate) withCorefileBase64(v string) (localDNSCorefileUpdate, error) { + decoded, err := base64.StdEncoding.DecodeString(strings.TrimSpace(v)) + if err != nil { + return localDNSCorefileUpdate{}, fmt.Errorf("decoding localDNS corefileBase64: %w", err) + } + if len(strings.TrimSpace(string(decoded))) == 0 { + return u, nil + } + u.corefile = string(decoded) + u.hasCorefile = true + return u, nil +} + +func firstNonEmpty(values ...string) string { + for _, v := range values { + if strings.TrimSpace(v) != "" { + return strings.TrimSpace(v) + } + } + return "" +} + +func (a *App) nodeAgentPoolName() (string, error) { + path := a.getNodeConfigPath() + raw, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("reading node config %s: %w", path, err) + } + cfg, perr := nodeconfigutils.UnmarshalConfigurationV1(raw) + if perr != nil { + slog.Info("node config parsed with errors, continuing with partial config", "error", perr) + } + if cfg == nil { + return "", fmt.Errorf("node config %s could not be parsed", path) + } + agentPool := cfg.GetKubeletConfig().GetKubeletNodeLabels()[localDNSAgentPoolLabel] + if agentPool == "" { + return "", fmt.Errorf("node config has no %s kubelet node label", localDNSAgentPoolLabel) + } + return agentPool, nil +} + +func (a *App) nodeConfigWithLocalDNSProfile(profile *aksnodeconfigv1.LocalDnsProfile) (*aksnodeconfigv1.Configuration, error) { + path := a.getNodeConfigPath() + raw, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("reading node config %s: %w", path, err) + } + cfg, perr := nodeconfigutils.UnmarshalConfigurationV1(raw) + if perr != nil { + slog.Info("node config parsed with errors, continuing with partial config", "error", perr) + } + if cfg == nil { + return nil, fmt.Errorf("node config %s could not be parsed", path) + } + cfg.LocalDnsProfile = profile + return cfg, nil +} + +func writeLocalDNSCorefile(path string, corefile string) error { + if strings.TrimSpace(corefile) == "" { + return fmt.Errorf("localDNS corefile is empty") + } + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return fmt.Errorf("create parent directory: %w", err) + } + tmp, err := os.CreateTemp(filepath.Dir(path), ".localdns-corefile-*") + if err != nil { + return fmt.Errorf("creating temp file: %w", err) + } + tmpPath := tmp.Name() + if _, err := io.WriteString(tmp, corefile); err != nil { + tmp.Close() + os.Remove(tmpPath) + return fmt.Errorf("writing temp file: %w", err) + } + if err := tmp.Close(); err != nil { + os.Remove(tmpPath) + return fmt.Errorf("closing temp file: %w", err) + } + if err := os.Chmod(tmpPath, 0644); err != nil { + os.Remove(tmpPath) + return fmt.Errorf("chmod temp file: %w", err) + } + if err := os.Rename(tmpPath, path); err != nil { + os.Remove(tmpPath) + return fmt.Errorf("renaming temp file: %w", err) + } + return nil +} + +func localDNSCorefileVersionPath(corefilePath string) string { + return corefilePath + ".version" +} + +func readLocalDNSCorefileVersion(path string) (string, error) { + data, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return "", nil + } + return "", fmt.Errorf("reading localDNS corefile version %s: %w", path, err) + } + return strings.TrimSpace(string(data)), nil +} + +func writeLocalDNSCorefileVersion(path string, version string) error { + if strings.TrimSpace(version) == "" { + return nil + } + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return fmt.Errorf("create parent directory: %w", err) + } + return os.WriteFile(path, []byte(strings.TrimSpace(version)+"\n"), 0600) +} diff --git a/aks-node-controller/localdnsconfig_test.go b/aks-node-controller/localdnsconfig_test.go new file mode 100644 index 00000000000..ddfbf2beefe --- /dev/null +++ b/aks-node-controller/localdnsconfig_test.go @@ -0,0 +1,221 @@ +package main + +import ( + "context" + "encoding/base64" + "fmt" + "net/http" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" +) + +func writeLocalDNSTestNodeConfig(t *testing.T, app *App) { + t.Helper() + p := filepath.Join(t.TempDir(), "aks-node-controller-config.json") + require.NoError(t, os.WriteFile(p, []byte(fmt.Sprintf(`{ + "version": "v1", + "kubelet_config": { + "kubelet_node_labels": { + "kubernetes.azure.com/agentpool": %q + } + } +}`, "pool1")), 0o600)) + app.nodeConfigPath = p +} + +func TestFetchAndApplyLocalDNSConfig(t *testing.T) { + t.Run("corefileBase64 rewrites output", func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + want := ".:53 {\n forward . 168.63.129.16\n}\n" + tt.App.fetchLocalDNSConfigFn = func(context.Context) (string, error) { + return `{"corefileBase64":"` + base64.StdEncoding.EncodeToString([]byte(want)) + `"}`, nil + } + out := filepath.Join(t.TempDir(), "localdns.corefile") + + outcome, err := tt.App.fetchAndApplyLocalDNSConfig(context.Background(), out) + require.NoError(t, err) + assert.Equal(t, outcomeLocalDNSConfigApplied, outcome) + got, err := os.ReadFile(out) + require.NoError(t, err) + assert.Equal(t, want, string(got)) + }) + + t.Run("agent pool corefileBase64 rewrites output", func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + writeLocalDNSTestNodeConfig(t, tt.App) + want := ".:53 {\n forward . 168.63.129.16\n reload\n}\n" + tt.App.fetchLocalDNSConfigFn = func(context.Context) (string, error) { + return `{"agentPools":{"pool1":{"corefileVersion":"abc123","corefileBase64":"` + + base64.StdEncoding.EncodeToString([]byte(want)) + `"},"pool2":{"corefileBase64":"ignored"}}}`, nil + } + out := filepath.Join(t.TempDir(), "localdns.corefile") + + outcome, err := tt.App.fetchAndApplyLocalDNSConfig(context.Background(), out) + require.NoError(t, err) + assert.Equal(t, outcomeLocalDNSConfigApplied, outcome) + got, err := os.ReadFile(out) + require.NoError(t, err) + assert.Equal(t, want, string(got)) + version, err := os.ReadFile(localDNSCorefileVersionPath(out)) + require.NoError(t, err) + assert.Equal(t, "abc123\n", string(version)) + }) + + t.Run("already current version and content skips rewrite", func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + writeLocalDNSTestNodeConfig(t, tt.App) + out := filepath.Join(t.TempDir(), "localdns.corefile") + original := ".:53 {\n forward . 1.1.1.1\n}\n" + require.NoError(t, os.WriteFile(out, []byte(original), 0o644)) + require.NoError(t, writeLocalDNSCorefileVersion(localDNSCorefileVersionPath(out), "abc123")) + tt.App.fetchLocalDNSConfigFn = func(context.Context) (string, error) { + return `{"agentPools":{"pool1":{"corefileVersion":"abc123","corefileBase64":"` + + base64.StdEncoding.EncodeToString([]byte(original)) + `"}}}`, nil + } + + outcome, err := tt.App.fetchAndApplyLocalDNSConfig(context.Background(), out) + require.NoError(t, err) + assert.Equal(t, outcomeLocalDNSConfigAlreadyCurrent, outcome) + got, err := os.ReadFile(out) + require.NoError(t, err) + assert.Equal(t, original, string(got)) + }) + + t.Run("matching version with stale corefile rewrites output", func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + writeLocalDNSTestNodeConfig(t, tt.App) + out := filepath.Join(t.TempDir(), "localdns.corefile") + require.NoError(t, os.WriteFile(out, []byte("stale-corefile"), 0o644)) + require.NoError(t, writeLocalDNSCorefileVersion(localDNSCorefileVersionPath(out), "abc123")) + want := ".:53 {\n forward . 168.63.129.16\n}\n" + tt.App.fetchLocalDNSConfigFn = func(context.Context) (string, error) { + return `{"agentPools":{"pool1":{"corefileVersion":"abc123","corefileBase64":"` + + base64.StdEncoding.EncodeToString([]byte(want)) + `"}}}`, nil + } + + outcome, err := tt.App.fetchAndApplyLocalDNSConfig(context.Background(), out) + require.NoError(t, err) + assert.Equal(t, outcomeLocalDNSConfigApplied, outcome) + got, err := os.ReadFile(out) + require.NoError(t, err) + assert.Equal(t, want, string(got)) + }) + + t.Run("agent pool version only config is no-op", func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + writeLocalDNSTestNodeConfig(t, tt.App) + tt.App.fetchLocalDNSConfigFn = func(context.Context) (string, error) { + return `{"agentPools":{"pool1":{"corefileVersion":"abc123"}}}`, nil + } + out := filepath.Join(t.TempDir(), "localdns.corefile") + + outcome, err := tt.App.fetchAndApplyLocalDNSConfig(context.Background(), out) + require.NoError(t, err) + assert.Equal(t, outcomeLocalDNSConfigNoCorefileData, outcome) + _, statErr := os.Stat(out) + assert.True(t, os.IsNotExist(statErr)) + }) + + t.Run("agent pool localDnsProfile renders corefile", func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + writeLocalDNSTestNodeConfig(t, tt.App) + tt.App.fetchLocalDNSConfigFn = func(context.Context) (string, error) { + return `{ + "agentPools": { + "pool1": { + "corefileVersion": "profile-hash", + "localDnsProfile": { + "enableLocalDns": true, + "vnetDnsOverrides": { + ".": { + "queryLogging": "Error", + "protocol": "PreferUDP", + "forwardDestination": "VnetDNS", + "forwardPolicy": "Sequential", + "maxConcurrent": 1000, + "cacheDurationInSeconds": 3600, + "serveStaleDurationInSeconds": 3600, + "serveStale": "Immediate" + } + }, + "kubeDnsOverrides": { + "cluster.local": { + "queryLogging": "Error", + "protocol": "PreferUDP", + "forwardDestination": "ClusterCoreDNS", + "forwardPolicy": "Sequential", + "maxConcurrent": 1000, + "cacheDurationInSeconds": 3600, + "serveStaleDurationInSeconds": 3600, + "serveStale": "Immediate" + } + } + } + } + } +}`, nil + } + out := filepath.Join(t.TempDir(), "localdns.corefile") + + outcome, err := tt.App.fetchAndApplyLocalDNSConfig(context.Background(), out) + require.NoError(t, err) + assert.Equal(t, outcomeLocalDNSConfigApplied, outcome) + got, err := os.ReadFile(out) + require.NoError(t, err) + assert.Contains(t, string(got), "health-check.localdns.local:53") + assert.Contains(t, string(got), "cluster.local:53") + version, err := os.ReadFile(localDNSCorefileVersionPath(out)) + require.NoError(t, err) + assert.Equal(t, "profile-hash\n", string(version)) + }) + + t.Run("other agent pool config is no-op", func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + writeLocalDNSTestNodeConfig(t, tt.App) + tt.App.fetchLocalDNSConfigFn = func(context.Context) (string, error) { + return `{"agentPools":{"pool2":{"corefileBase64":"` + base64.StdEncoding.EncodeToString([]byte("ignored")) + `"}}}`, nil + } + out := filepath.Join(t.TempDir(), "localdns.corefile") + + outcome, err := tt.App.fetchAndApplyLocalDNSConfig(context.Background(), out) + require.NoError(t, err) + assert.Equal(t, outcomeLocalDNSConfigNoCorefileData, outcome) + _, statErr := os.Stat(out) + assert.True(t, os.IsNotExist(statErr)) + }) + + t.Run("cli action fails open", func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + tt.App.fetchLocalDNSConfigFn = func(context.Context) (string, error) { + return "", assert.AnError + } + exitCode := tt.App.Run(context.Background(), []string{"aks-node-controller", "fetch-localdns-config", "--output", filepath.Join(t.TempDir(), "localdns.corefile")}) + assert.Equal(t, 0, exitCode) + }) +} + +func TestLocalDNSLPSUnavailableStatusCode(t *testing.T) { + tests := []struct { + name string + code codes.Code + statusCode int + ok bool + }{ + {name: "not found", code: codes.NotFound, statusCode: http.StatusNotFound, ok: true}, + {name: "permission denied", code: codes.PermissionDenied, statusCode: http.StatusForbidden, ok: true}, + {name: "unauthenticated", code: codes.Unauthenticated, statusCode: http.StatusUnauthorized, ok: true}, + {name: "internal", code: codes.Internal, ok: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + statusCode, ok := localDNSLPSUnavailableStatusCode(tt.code) + assert.Equal(t, tt.ok, ok) + assert.Equal(t, tt.statusCode, statusCode) + }) + } +} diff --git a/aks-node-controller/parser/helper.go b/aks-node-controller/parser/helper.go index 9cc9e7eecca..0f348a40f42 100644 --- a/aks-node-controller/parser/helper.go +++ b/aks-node-controller/parser/helper.go @@ -894,7 +894,7 @@ type localDnsCorefileTemplateData struct { // Corefile is created using localdns.toml.gtpl template and aksnodeconfig values. // includeHostsPlugin controls whether the hosts plugin block is included in the generated Corefile. -func generateLocalDnsCorefileFromAKSNodeConfig(aksnodeconfig *aksnodeconfigv1.Configuration, includeHostsPlugin bool) (string, error) { +func GenerateLocalDNSCorefileFromAKSNodeConfig(aksnodeconfig *aksnodeconfigv1.Configuration, includeHostsPlugin bool) (string, error) { var corefileBuffer bytes.Buffer templateData := localDnsCorefileTemplateData{ Config: aksnodeconfig, @@ -906,6 +906,10 @@ func generateLocalDnsCorefileFromAKSNodeConfig(aksnodeconfig *aksnodeconfigv1.Co return corefileBuffer.String(), nil } +func generateLocalDnsCorefileFromAKSNodeConfig(aksnodeconfig *aksnodeconfigv1.Configuration, includeHostsPlugin bool) (string, error) { + return GenerateLocalDNSCorefileFromAKSNodeConfig(aksnodeconfig, includeHostsPlugin) +} + // getLocalDnsClusterListenerIp returns APIPA-IP address that will be used in localdns systemd unit. func getLocalDnsClusterListenerIp() string { return localDnsClusterListenerIp diff --git a/aks-node-controller/parser/helper_test.go b/aks-node-controller/parser/helper_test.go index 567431d225b..65382dbd732 100644 --- a/aks-node-controller/parser/helper_test.go +++ b/aks-node-controller/parser/helper_test.go @@ -1687,6 +1687,7 @@ health-check.localdns.local:53 { max_concurrent 1000 } ready 169.254.10.10:8181 + reload cache 3600 { success 9984 denial 9984 @@ -1714,6 +1715,7 @@ cluster.local:53 { max_concurrent 1000 } ready 169.254.10.10:8181 + reload cache 3600 { success 9984 denial 9984 @@ -1731,6 +1733,7 @@ testdomain456.com:53 { max_concurrent 1000 } ready 169.254.10.10:8181 + reload cache 3600 { success 9984 denial 9984 @@ -1756,6 +1759,7 @@ testdomain456.com:53 { max_concurrent 2000 } ready 169.254.10.11:8181 + reload cache 3600 { success 9984 denial 9984 diff --git a/aks-node-controller/parser/templates/localdns.toml.gtpl b/aks-node-controller/parser/templates/localdns.toml.gtpl index 818b23aa421..191cee3cbcb 100644 --- a/aks-node-controller/parser/templates/localdns.toml.gtpl +++ b/aks-node-controller/parser/templates/localdns.toml.gtpl @@ -47,6 +47,7 @@ health-check.localdns.local:53 { max_concurrent {{$override.MaxConcurrent}} } ready {{getLocalDnsNodeListenerIp}}:8181 + reload cache {{$override.CacheDurationInSeconds}} { success 9984 denial 9984 @@ -112,6 +113,7 @@ health-check.localdns.local:53 { max_concurrent {{$override.MaxConcurrent}} } ready {{getLocalDnsClusterListenerIp}}:8181 + reload cache {{$override.CacheDurationInSeconds}} { success 9984 denial 9984 diff --git a/e2e/scenario_localdns_hosts_test.go b/e2e/scenario_localdns_hosts_test.go index c83eba792a9..32b598c7df6 100644 --- a/e2e/scenario_localdns_hosts_test.go +++ b/e2e/scenario_localdns_hosts_test.go @@ -1,12 +1,30 @@ package e2e import ( + "bytes" + "compress/gzip" + "context" + "encoding/base64" + "os" + "strings" "testing" + "time" aksnodeconfigv1 "github.com/Azure/agentbaker/aks-node-controller/pkg/gen/aksnodeconfig/v1" "github.com/Azure/agentbaker/e2e/config" "github.com/Azure/agentbaker/pkg/agent/datamodel" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v7" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" +) + +const ( + desiredLocalDNSVersion = "e2e-localdns-corefile-version" + localDNSPayloadPath = "/opt/azure/containers/localdns/e2e-localdns-lps-payload.json" + localDNSFetcherStamp = "/opt/azure/containers/localdns/e2e-localdns-lps-fetcher-called" + localDNSBranchScriptArchivePath = "/opt/azure/containers/localdns/e2e-localdns.sh.gz.b64" + localDNSFetcherPath = "/opt/azure/containers/localdns/e2e-fetch-localdns-config" ) // Test_LocalDNSHostsPlugin tests the localdns hosts plugin across all supported distros @@ -56,3 +74,167 @@ func Test_LocalDNSHostsPlugin(t *testing.T) { }) } } + +// Test_LocalDNSLPSBootstrapPatch validates the node-side LocalDNS live-patching +// bootstrap path. It simulates LPS by temporarily wrapping aks-node-controller's +// fetch-localdns-config command so it returns a LocalDNS nodeConfig payload, then +// delegates to the real apply-localdns-config implementation. The test verifies that +// localdns.sh: +// 1. invokes the fetcher before CoreDNS starts, +// 2. renders the supplied LocalDNS profile payload into updated.localdns.corefile, +// 3. persists the paired corefileVersion, and +// 4. stamps components.localDNS.current after kubeconfig/node registration. +func Test_LocalDNSLPSBootstrapPatch(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests LocalDNS LPS bootstrap patching applies Corefile and reports corefileVersion", + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2404Gen2Containerd, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.AgentPoolProfile.LocalDNSProfile.EnableLocalDNS = true + }, + CustomDataWriteFiles: []CustomDataWriteFile{ + { + Path: localDNSBranchScriptArchivePath, + Permissions: "0644", + Owner: "root", + Content: mustReadCompressedLocalDNSArtifact(t), + }, + { + Path: "/etc/systemd/system/localdns.service.d/00-e2e-branch-localdns.conf", + Permissions: "0644", + Owner: "root", + Content: localDNSBranchScriptDropIn(), + }, + { + Path: localDNSPayloadPath, + Permissions: "0644", + Owner: "root", + Content: localDNSLPSPayload(desiredLocalDNSVersion), + }, + { + Path: localDNSFetcherPath, + Permissions: "0755", + Owner: "root", + Content: localDNSLPSFetcherWrapper(), + }, + }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) { + config.LocalDnsProfile.EnableLocalDns = true + }, + Validator: validateLocalDNSLPSBootstrapPatch, + }, + }) +} + +func mustReadCompressedLocalDNSArtifact(t *testing.T) string { + t.Helper() + data, err := os.ReadFile("../parts/linux/cloud-init/artifacts/localdns.sh") + require.NoError(t, err) + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + _, err = zw.Write(data) + require.NoError(t, err) + require.NoError(t, zw.Close()) + content := strings.ReplaceAll(string(data), `AKS_NODE_CONTROLLER_BINARY="/opt/azure/containers/aks-node-controller"`, `AKS_NODE_CONTROLLER_BINARY="`+localDNSFetcherPath+`"`) + buf.Reset() + zw = gzip.NewWriter(&buf) + _, err = zw.Write([]byte(content)) + require.NoError(t, err) + require.NoError(t, zw.Close()) + return base64.StdEncoding.EncodeToString(buf.Bytes()) +} + +func localDNSBranchScriptDropIn() string { + return `[Service] +ExecStartPre=/bin/bash -c 'base64 -d ` + localDNSBranchScriptArchivePath + ` | gzip -d > /opt/azure/containers/localdns.sh && chmod 0544 /opt/azure/containers/localdns.sh' +` +} + +func localDNSLPSPayload(version string) string { + return `{ + "agentPools": { + "nodepool2": { + "corefileVersion": "` + version + `", + "localDnsProfile": { + "enableLocalDns": true, + "vnetDnsOverrides": { + ".": { + "queryLogging": "Error", + "protocol": "PreferUDP", + "forwardDestination": "VnetDNS", + "forwardPolicy": "Sequential", + "maxConcurrent": 1000, + "cacheDurationInSeconds": 3600, + "serveStaleDurationInSeconds": 3600, + "serveStale": "Immediate" + } + }, + "kubeDnsOverrides": { + "cluster.local": { + "queryLogging": "Error", + "protocol": "PreferUDP", + "forwardDestination": "ClusterCoreDNS", + "forwardPolicy": "Sequential", + "maxConcurrent": 1000, + "cacheDurationInSeconds": 3600, + "serveStaleDurationInSeconds": 3600, + "serveStale": "Immediate" + } + } + } + } + } +}` +} + +func localDNSLPSFetcherWrapper() string { + return `#!/bin/bash +set -euo pipefail +if [ "${1:-}" != "fetch-localdns-config" ]; then + exec /opt/azure/containers/aks-node-controller "$@" +fi +output="" +while [ "$#" -gt 0 ]; do + case "$1" in + --output) + output="$2" + shift 2 + ;; + *) + shift + ;; + esac +done +if [ -z "$output" ]; then + echo "missing --output" >&2 + exit 1 +fi +touch ` + localDNSFetcherStamp + ` +exec /opt/azure/containers/aks-node-controller apply-localdns-config --config-file ` + localDNSPayloadPath + ` --output "$output" +` +} + +func validateLocalDNSLPSBootstrapPatch(ctx context.Context, s *Scenario) { + const ( + updatedCorefile = "/opt/azure/containers/localdns/updated.localdns.corefile" + livepatchedCorefile = "/opt/azure/containers/localdns/livepatched.localdns.corefile" + ) + + ValidateFileExists(ctx, s, localDNSFetcherStamp) + ValidateFileHasContent(ctx, s, updatedCorefile, "health-check.localdns.local:53") + ValidateFileHasContent(ctx, s, updatedCorefile, "cluster.local:53") + ValidateFileHasContent(ctx, s, livepatchedCorefile+".version", desiredLocalDNSVersion) + ValidateLocalDNSService(ctx, s, "enabled") + ValidateLocalDNSResolution(ctx, s, "169.254.10.10") + + err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 3*time.Minute, true, func(ctx context.Context) (bool, error) { + node, err := s.Runtime.Kube.Typed.CoreV1().Nodes().Get(ctx, s.Runtime.VM.KubeName, metav1.GetOptions{}) + if err != nil { + return false, nil + } + status := node.Annotations["kubernetes.azure.com/live-patching-status"] + return strings.Contains(status, `"localDNS":{"current":"`+desiredLocalDNSVersion+`"}`), nil + }) + require.NoError(s.T, err, "node did not report LocalDNS live-patching current version %q", desiredLocalDNSVersion) +} diff --git a/parts/linux/cloud-init/artifacts/localdns.sh b/parts/linux/cloud-init/artifacts/localdns.sh index 6f07c1ee455..0156e276201 100644 --- a/parts/linux/cloud-init/artifacts/localdns.sh +++ b/parts/linux/cloud-init/artifacts/localdns.sh @@ -24,6 +24,8 @@ LOCALDNS_CGROUP_DIR="/sys/fs/cgroup/localdns.slice/localdns.service" LOCALDNS_CORE_FILE="${LOCALDNS_SCRIPT_PATH}/localdns.corefile" # This is the localdns corefile that has updated UpstreamDNSServerIPs and will be used by the localdns systemd unit. +LIVEPATCHED_LOCALDNS_CORE_FILE="${LOCALDNS_SCRIPT_PATH}/livepatched.localdns.corefile" + UPDATED_LOCALDNS_CORE_FILE="${LOCALDNS_SCRIPT_PATH}/updated.localdns.corefile" # This is slice file used by localdns systemd unit. @@ -69,6 +71,10 @@ START_LOCALDNS_TIMEOUT=10 DNS_HEALTH_CHECK_TIMEOUT=2 DNS_HEALTH_CHECK_TRIES=2 +AKS_NODE_CONTROLLER_BINARY="/opt/azure/containers/aks-node-controller" +LOCALDNS_LIVE_PATCHING_COMPONENT_NAME="localDNS" +LOCALDNS_LIVE_PATCHING_STATUS_ANNOTATION="kubernetes.azure.com/live-patching-status" + # Function definitions used in this file. # functions defined until "${__SOURCED__:+return}" are sourced and tested in - # spec/parts/linux/cloud-init/artifacts/localdns_spec.sh. @@ -157,6 +163,14 @@ regenerate_localdns_corefile() { return 0 } +localdns_source_corefile() { + if [ -s "${LIVEPATCHED_LOCALDNS_CORE_FILE:-}" ]; then + echo "${LIVEPATCHED_LOCALDNS_CORE_FILE}" + return 0 + fi + echo "${LOCALDNS_CORE_FILE}" +} + # Replace AzureDNSIP in corefile with VNET DNS ServerIPs if necessary. replace_azurednsip_in_corefile() { if [ -z "${RESOLV_CONF:-}" ]; then @@ -187,8 +201,10 @@ replace_azurednsip_in_corefile() { # and also not equal to the localdns node listener IP to avoid creating a circular dependency. # Corefile will have 168.63.129.16 when user input has VnetDNS value for forwarddestination. # Note - For root domain under VnetDNSOverrides, all DNS traffic should be forwarded to VnetDNS. - cp "${LOCALDNS_CORE_FILE}" "${UPDATED_LOCALDNS_CORE_FILE}" || { - echo "Failed to copy ${LOCALDNS_CORE_FILE} to ${UPDATED_LOCALDNS_CORE_FILE}" + local source_corefile + source_corefile="$(localdns_source_corefile)" + cp "${source_corefile}" "${UPDATED_LOCALDNS_CORE_FILE}" || { + echo "Failed to copy ${source_corefile} to ${UPDATED_LOCALDNS_CORE_FILE}" return 1 } @@ -304,6 +320,119 @@ replace_azurednsip_in_corefile() { return 0 } +refresh_localdns_corefile_from_lps() { + if [ -z "${LIVEPATCHED_LOCALDNS_CORE_FILE:-}" ]; then + echo "LIVEPATCHED_LOCALDNS_CORE_FILE is not set or is empty." + return 1 + fi + + if [ ! -x "${AKS_NODE_CONTROLLER_BINARY}" ]; then + echo "AKS node controller binary not found at ${AKS_NODE_CONTROLLER_BINARY}; skipping LocalDNS LPS config fetch." + return 0 + fi + + # Write the LPS-provided Corefile to the livepatched source file; VNET DNS replacement + # later derives UPDATED_LOCALDNS_CORE_FILE from this file before CoreDNS starts. + if "${AKS_NODE_CONTROLLER_BINARY}" fetch-localdns-config --output "${LIVEPATCHED_LOCALDNS_CORE_FILE}"; then + echo "Completed LocalDNS LPS config fetch." + return 0 + fi + + echo "LocalDNS LPS config fetch failed; continuing with existing corefile." + return 0 +} + +localdns_corefile_version_file() { + echo "${LIVEPATCHED_LOCALDNS_CORE_FILE}.version" +} + +wait_for_kubeconfig_and_node() { + if [ ! -x /opt/bin/kubectl ]; then + echo "kubectl binary not found at /opt/bin/kubectl, skipping annotation." >&2 + return 1 + fi + + local kubeconfig="${KUBECONFIG:-/var/lib/kubelet/kubeconfig}" + local wait_count=0 + local max_wait="${KUBECONFIG_WAIT_ATTEMPTS:-60}" + while [ ! -f "${kubeconfig}" ]; do + if [ $wait_count -ge $max_wait ]; then + echo "Timeout waiting for kubeconfig at ${kubeconfig} after ${max_wait} attempts, skipping annotation." >&2 + return 1 + fi + echo "Waiting for TLS bootstrapping to complete (attempt $((wait_count + 1))/${max_wait})..." >&2 + sleep 3 + wait_count=$((wait_count + 1)) + done + echo "Kubeconfig found at ${kubeconfig}" >&2 + + local node_name + node_name=$(hostname) + if [ -z "${node_name}" ]; then + echo "Cannot get node name, skipping annotation." >&2 + return 1 + fi + node_name=$(echo "$node_name" | tr '[:upper:]' '[:lower:]') + + echo "Waiting for node ${node_name} to be registered in the cluster..." >&2 + local node_wait_count=0 + local max_node_wait="${NODE_REGISTRATION_WAIT_ATTEMPTS:-30}" + while [ $node_wait_count -lt $max_node_wait ]; do + if /opt/bin/kubectl --kubeconfig "${kubeconfig}" get node "${node_name}" >/dev/null 2>&1; then + echo "${kubeconfig}|${node_name}" + return 0 + fi + echo "Waiting for node registration (attempt $((node_wait_count + 1))/${max_node_wait})..." >&2 + sleep 3 + node_wait_count=$((node_wait_count + 1)) + done + + echo "Timeout waiting for node ${node_name} to be registered after ${max_node_wait} attempts, skipping annotation." >&2 + return 1 +} + +annotate_node_with_localdns_livepatch_status() { + local version_file + version_file="$(localdns_corefile_version_file)" + if [ ! -s "${version_file}" ]; then + echo "LocalDNS corefile version file not found at ${version_file}, skipping live patching status annotation." + return 0 + fi + + local corefile_version + corefile_version="$(tr -d '[:space:]' < "${version_file}")" + if [ -z "${corefile_version}" ]; then + echo "LocalDNS corefile version file is empty, skipping live patching status annotation." + return 0 + fi + + local kube_node + kube_node="$(wait_for_kubeconfig_and_node)" || return 0 + local kubeconfig="${kube_node%%|*}" + local node_name="${kube_node#*|}" + local current_status + current_status=$(/opt/bin/kubectl --kubeconfig "${kubeconfig}" get node "${node_name}" -o "jsonpath={.metadata.annotations['kubernetes\.azure\.com/live-patching-status']}" 2>/dev/null || true) + if [ -z "${current_status}" ]; then + current_status='{}' + fi + + local updated_status + if ! updated_status="$(printf '%s' "${current_status}" | jq -c \ + --arg component "${LOCALDNS_LIVE_PATCHING_COMPONENT_NAME}" \ + --arg current "${corefile_version}" \ + '.components = (.components // {}) | .components[$component].current = $current')"; then + echo "Failed to render LocalDNS live patching status annotation." + return 0 + fi + + echo "Setting LocalDNS live patching current version ${corefile_version} for node ${node_name}." + if /opt/bin/kubectl --kubeconfig "${kubeconfig}" annotate --overwrite node "${node_name}" "${LOCALDNS_LIVE_PATCHING_STATUS_ANNOTATION}=${updated_status}"; then + echo "Successfully set LocalDNS live patching status annotation." + else + echo "Warning: Failed to set LocalDNS live patching status annotation (this is non-fatal)." + fi +} + # Build iptables rules to skip conntrack for DNS traffic to localdns. build_localdns_iptable_rules() { # These rules skip conntrack for DNS traffic to the local DNS service IPs to save conntrack table space. @@ -1026,6 +1155,12 @@ if ! wait_for_localdns_removed_from_resolv_conf 5; then exit $ERR_LOCALDNS_FAIL fi +# Fetch LocalDNS config from LPS if present. This is fail-open: no config or fetch errors keep the +# locally generated corefile. If LPS returns a usable profile/corefile, LIVEPATCHED_LOCALDNS_CORE_FILE +# is written before VNET DNS replacement builds UPDATED_LOCALDNS_CORE_FILE for CoreDNS. +# --------------------------------------------------------------------------------------------------------------------- +refresh_localdns_corefile_from_lps + # Replace AzureDNSIP in corefile with VNET DNS ServerIPs. # --------------------------------------------------------------------------------------------------------------------- replace_azurednsip_in_corefile || exit $ERR_LOCALDNS_FAIL @@ -1073,6 +1208,17 @@ echo "Startup complete - serving node and pod DNS traffic." # Export initial resource metrics so the exporter has data before the first watchdog tick. export_resource_metrics +# The generic knead live-patching loop owns kubernetes.azure.com/live-patching-status at runtime. +# Keep this legacy/bootstrap writer opt-in to avoid racing knead's status update. +# -------------------------------------------------------------------------------------------------------------------- +if [ "${LOCALDNS_ENABLE_LEGACY_LIVEPATCH_STATUS:-false}" = "true" ]; then + annotate_node_with_localdns_livepatch_status & + LOCALDNS_LIVEPATCH_ANNOTATION_PID=$! + echo "Started LocalDNS live-patching status annotation in background (PID: ${LOCALDNS_LIVEPATCH_ANNOTATION_PID})" +else + echo "Skipping LocalDNS live-patching status annotation; knead owns live-patching-status." +fi + # Set node annotation to indicate hosts plugin is in use (if applicable). # -------------------------------------------------------------------------------------------------------------------- # Only run when hosts plugin is currently enabled or was previously enabled (marker exists). diff --git a/parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh b/parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh index b2e986a2d41..a23e4c80b37 100755 --- a/parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh +++ b/parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh @@ -237,6 +237,10 @@ knead_apply_components() { component_comparator=securityPatchIsCurrent component_handler=updateSecurityPatch ;; + localDNS) + component_comparator=localDNSIsCurrent + component_handler=updateLocalDNS + ;; *) echo "unsupported component: ${component}" component_index=$((component_index + 1)) @@ -278,6 +282,57 @@ knead_apply_components() { } # Records the processed hash and per-component results in the node status annotation. +localDNSIsCurrent() { + local desired_payload="$1" + local current_payload="$2" + + [ "${desired_payload}" = "${current_payload}" ] +} + +updateLocalDNS() { + local component_payload="${1:-}" + local outcome + + if [ ! -x /opt/azure/containers/aks-node-controller ]; then + echo "aks-node-controller binary is required for localDNS live patching" + return 1 + fi + + if ! outcome="$(printf '%s' "${component_payload}" | /opt/azure/containers/aks-node-controller apply-localdns-config \ + --config-file - \ + --output /opt/azure/containers/localdns/livepatched.localdns.corefile)"; then + echo "localDNS config apply failed" + return 1 + fi + printf '%s +' "${outcome}" + + case "$(printf '%s +' "${outcome}" | tail -n 1)" in + applied) + if ! systemctl restart localdns.service; then + echo "failed to restart localdns.service" + return 1 + fi + echo "localDNS update completed successfully" + ;; + alreadyCurrent) + echo "localDNS is already current" + ;; + notFound) + echo "localDNS LPS config is not available" + ;; + noCorefileData) + echo "localDNS LPS config has no node-applicable payload" + return 1 + ;; + *) + echo "unexpected localDNS apply outcome: ${outcome}" + return 1 + ;; + esac +} + knead_write_status() { local node_name="$1" local goal="$2" diff --git a/pkg/agent/baker.go b/pkg/agent/baker.go index 873790f8eff..f0801a0b70a 100644 --- a/pkg/agent/baker.go +++ b/pkg/agent/baker.go @@ -2310,6 +2310,7 @@ health-check.localdns.local:53 { max_concurrent {{$override.MaxConcurrent}} } ready {{$.NodeListenerIP}}:8181 + reload cache {{$override.CacheDurationInSeconds}} { success 9984 denial 9984 @@ -2375,6 +2376,7 @@ health-check.localdns.local:53 { max_concurrent {{$override.MaxConcurrent}} } ready {{$.ClusterListenerIP}}:8181 + reload cache {{$override.CacheDurationInSeconds}} { success 9984 denial 9984 diff --git a/pkg/agent/baker_test.go b/pkg/agent/baker_test.go index 9ea69ecdc4e..67c4a2a7adc 100644 --- a/pkg/agent/baker_test.go +++ b/pkg/agent/baker_test.go @@ -474,6 +474,7 @@ health-check.localdns.local:53 { max_concurrent 1000 } ready 169.254.10.10:8181 + reload cache 3600 { success 9984 denial 9984 @@ -501,6 +502,7 @@ cluster.local:53 { max_concurrent 1000 } ready 169.254.10.10:8181 + reload cache 3600 { success 9984 denial 9984 @@ -518,6 +520,7 @@ testdomain456.com:53 { max_concurrent 1000 } ready 169.254.10.10:8181 + reload cache 3600 { success 9984 denial 9984 @@ -543,6 +546,7 @@ testdomain456.com:53 { max_concurrent 2000 } ready 169.254.10.11:8181 + reload cache 3600 { success 9984 denial 9984 @@ -663,6 +667,7 @@ health-check.localdns.local:53 { max_concurrent 1000 } ready 169.254.10.10:8181 + reload cache 3600 { success 9984 denial 9984 @@ -690,6 +695,7 @@ cluster.local:53 { max_concurrent 1000 } ready 169.254.10.10:8181 + reload cache 3600 { success 9984 denial 9984 @@ -707,6 +713,7 @@ testdomain456.com:53 { max_concurrent 1000 } ready 169.254.10.10:8181 + reload cache 3600 { success 9984 denial 9984 @@ -732,6 +739,7 @@ testdomain456.com:53 { max_concurrent 1000 } ready 169.254.10.11:8181 + reload cache 3600 { success 9984 denial 9984 @@ -759,6 +767,7 @@ cluster.local:53 { max_concurrent 1000 } ready 169.254.10.11:8181 + reload cache 3600 { success 9984 denial 9984 @@ -776,6 +785,7 @@ testdomain567.com:53 { max_concurrent 1000 } ready 169.254.10.11:8181 + reload cache 3600 { success 9984 denial 9984 diff --git a/spec/parts/linux/cloud-init/artifacts/localdns_spec.sh b/spec/parts/linux/cloud-init/artifacts/localdns_spec.sh index 65fbaae2cb9..6b061521fe2 100644 --- a/spec/parts/linux/cloud-init/artifacts/localdns_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/localdns_spec.sh @@ -23,6 +23,7 @@ Describe 'localdns.sh' TEST_DIR="/tmp/localdnstest" LOCALDNS_SCRIPT_PATH="${TEST_DIR}/opt/azure/containers/localdns" LOCALDNS_CORE_FILE="${LOCALDNS_SCRIPT_PATH}/localdns.corefile" + LIVEPATCHED_LOCALDNS_CORE_FILE="${LOCALDNS_SCRIPT_PATH}/livepatched.localdns.corefile" UPDATED_LOCALDNS_CORE_FILE="${LOCALDNS_SCRIPT_PATH}/updated.localdns.corefile" mkdir -p "$LOCALDNS_SCRIPT_PATH" # Use production-realistic corefile format with brace syntax @@ -391,6 +392,34 @@ EOF The stdout should include "Successfully exported forward IPs to ${LOCALDNS_SCRIPT_PATH}/forward_ips.prom" End + It 'should fetch LocalDNS LPS config through aks-node-controller when binary exists' + AKS_NODE_CONTROLLER_BINARY="${TEST_DIR}/aks-node-controller" + cat > "${AKS_NODE_CONTROLLER_BINARY}" <<'EOF' +#!/bin/bash +echo "anc args: $*" +exit 0 +EOF + chmod +x "${AKS_NODE_CONTROLLER_BINARY}" + + When run refresh_localdns_corefile_from_lps + The status should be success + The output should include "anc args: fetch-localdns-config --output ${LIVEPATCHED_LOCALDNS_CORE_FILE}" + The output should include "Completed LocalDNS LPS config fetch." + End + + It 'should skip LocalDNS LPS config fetch when aks-node-controller binary is missing' + AKS_NODE_CONTROLLER_BINARY="${TEST_DIR}/missing-aks-node-controller" + When run refresh_localdns_corefile_from_lps + The status should be success + The output should include "AKS node controller binary not found at ${AKS_NODE_CONTROLLER_BINARY}; skipping LocalDNS LPS config fetch." + End + + It 'should skip LocalDNS live patching status annotation when version file is missing' + When run annotate_node_with_localdns_livepatch_status + The status should be success + The output should include "LocalDNS corefile version file not found at ${LIVEPATCHED_LOCALDNS_CORE_FILE}.version, skipping live patching status annotation." + End + It 'should set correct permissions on forward_ips.prom file' When run replace_azurednsip_in_corefile The status should be success @@ -1960,3 +1989,50 @@ KUBECTL_EOF End End End + + + Describe 'livepatched corefile source selection' + setup() { + Include "./parts/linux/cloud-init/artifacts/localdns.sh" + TEST_DIR="/tmp/localdns-livepatched-test" + LOCALDNS_SCRIPT_PATH="${TEST_DIR}/opt/azure/containers/localdns" + LOCALDNS_CORE_FILE="${LOCALDNS_SCRIPT_PATH}/localdns.corefile" + LIVEPATCHED_LOCALDNS_CORE_FILE="${LOCALDNS_SCRIPT_PATH}/livepatched.localdns.corefile" + UPDATED_LOCALDNS_CORE_FILE="${LOCALDNS_SCRIPT_PATH}/updated.localdns.corefile" + RESOLV_CONF="${TEST_DIR}/run/systemd/resolve/resolv.conf" + mkdir -p "${LOCALDNS_SCRIPT_PATH}" "$(dirname "${RESOLV_CONF}")" + echo 'nameserver 10.0.0.1' > "${RESOLV_CONF}" + } + cleanup() { + rm -rf "${TEST_DIR}" + } + BeforeEach 'setup' + AfterEach 'cleanup' + + It 'uses the livepatched Corefile when present' + printf '.:53 { + forward . 9.9.9.9 +} +' > "${LOCALDNS_CORE_FILE}" + printf '.:53 { + forward . 168.63.129.16 +} +' > "${LIVEPATCHED_LOCALDNS_CORE_FILE}" + When run replace_azurednsip_in_corefile + The status should be success + The output should include 'Successfully updated' + The contents of file "${UPDATED_LOCALDNS_CORE_FILE}" should include '10.0.0.1' + The contents of file "${UPDATED_LOCALDNS_CORE_FILE}" should not include '9.9.9.9' + End + + It 'falls back to the generated Corefile when no livepatched Corefile exists' + printf '.:53 { + forward . 168.63.129.16 +} +' > "${LOCALDNS_CORE_FILE}" + When run replace_azurednsip_in_corefile + The status should be success + The output should include 'Successfully updated' + The contents of file "${UPDATED_LOCALDNS_CORE_FILE}" should include '10.0.0.1' + End + End diff --git a/spec/parts/linux/cloud-init/artifacts/ubuntu-snapshot-update_spec.sh b/spec/parts/linux/cloud-init/artifacts/ubuntu-snapshot-update_spec.sh index 78692c3f109..07e748a00f3 100644 --- a/spec/parts/linux/cloud-init/artifacts/ubuntu-snapshot-update_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/ubuntu-snapshot-update_spec.sh @@ -21,9 +21,10 @@ Describe 'ubuntu-snapshot-update.sh generic reconciliation' TEST_REPO_SERVICE="" printf '%s' '{"components":[]}' > "${TEST_COMPONENTS_JSON_FILE}" TEST_SECURITY_STATUS=0 + TEST_LOCALDNS_STATUS=0 TEST_ANNOTATE_STATUS=0 export KUBECTL KNEAD_COMPONENT_STATE_FILE TEST_COMPONENTS_JSON_FILE TEST_KUBECTL_ARGS_FILE - export TEST_STATUS TEST_GOAL TEST_AGENT_POOL TEST_REPO_SERVICE TEST_SECURITY_STATUS TEST_ANNOTATE_STATUS + export TEST_STATUS TEST_GOAL TEST_AGENT_POOL TEST_REPO_SERVICE TEST_SECURITY_STATUS TEST_LOCALDNS_STATUS TEST_ANNOTATE_STATUS } cleanup() { @@ -66,6 +67,12 @@ Describe 'ubuntu-snapshot-update.sh generic reconciliation' return "${TEST_SECURITY_STATUS}" } + + updateLocalDNS() { + echo "updateLocalDNS called with args: $*" + return "${TEST_LOCALDNS_STATUS}" + } + securityPatchIsCurrent() { local desired_payload="$1" local current_payload="$2" @@ -157,6 +164,54 @@ Describe 'ubuntu-snapshot-update.sh generic reconciliation' The contents of file "${TEST_KUBECTL_ARGS_FILE}" should equal 'get cm -n kube-system live-patching-config -o jsonpath={.data.live-patching-config\.json}' End + + It 'dispatches localDNS and writes successful status' + mkdir -p /opt/azure/containers + cat > /opt/azure/containers/aks-node-controller <<'EOF' +#!/bin/bash +echo "aks-node-controller called with args: $*" +cat > /tmp/localdns-livepatch-payload +echo applied +EOF + chmod +x /opt/azure/containers/aks-node-controller + Mock systemctl + echo "systemctl called with args: $*" + End + set_payload_goal '{"components":[{"name":"localDNS","nodeConfig":"{\"profiles\":{\"ap1\":{\"configChecksum\":\"localdns-v1\"}}}"}]}' + + When call knead_main + The status should be success + The output should include 'applying component: localDNS' + The output should include 'aks-node-controller called with args: apply-localdns-config --config-file - --output /opt/azure/containers/localdns/livepatched.localdns.corefile' + The contents of file "/tmp/localdns-livepatch-payload" should include '"configChecksum":"localdns-v1"' + The output should include 'systemctl called with args: restart localdns.service' + The output should include 'localDNS update completed successfully' + The output should include 'annotate mock called with args: annotate --overwrite node aks-node-1 kubernetes.azure.com/live-patching-status={"currentHash":"' + The output should include '"components":{"localDNS":{"code":"Succeeded"}}}' + The contents of file "${KNEAD_COMPONENT_STATE_FILE}" should include '"localDNS"' + End + + It 'marks localDNS failed when service restart fails' + mkdir -p /opt/azure/containers + cat > /opt/azure/containers/aks-node-controller <<'EOF' +#!/bin/bash +echo applied +EOF + chmod +x /opt/azure/containers/aks-node-controller + Mock systemctl + echo "systemctl called with args: $*" + exit 1 + End + set_payload_goal '{"components":[{"name":"localDNS","nodeConfig":"{\"profiles\":{\"ap1\":{\"configChecksum\":\"localdns-v1\"}}}"}]}' + + When call knead_main + The status should be failure + The output should include 'applying component: localDNS' + The output should include 'failed to restart localdns.service' + The output should include 'component failed: localDNS' + The output should include '"components":{"localDNS":{"code":"Failed"}}}' + End + It 'fails before dispatch when the goal hash does not match the ConfigMap payload' printf '%s' '{"components":[{"name":"securityPatch","nodeConfig":"{\"agentPools\":{}}"}]}' > "${TEST_COMPONENTS_JSON_FILE}" TEST_GOAL="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" From 42d67e20048068b368e765794bf24d0df7ddde47 Mon Sep 17 00:00:00 2001 From: Saewon Kwak <23280628+saewoni@users.noreply.github.com> Date: Tue, 18 Aug 2026 23:44:16 +0000 Subject: [PATCH 29/38] Refresh PR after clean rebuild Force GitHub to rebuild the stale PR merge ref after replacing the stacked branch with a clean LocalDNS commit on top of feature/knead-security-patching. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> From f06f8d918bec801807ea0709f006885107723ada Mon Sep 17 00:00:00 2001 From: Saewon Kwak <23280628+saewoni@users.noreply.github.com> Date: Fri, 21 Aug 2026 20:01:29 +0000 Subject: [PATCH 30/38] Preserve boothook launcher when injecting E2E files Insert e2e CustomDataWriteFiles before the full aks-node-controller launcher line instead of inside the nohup command, and add tests covering the nohup and systemd launcher cases. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- e2e/vmss.go | 20 ++++++++++++++++---- e2e/vmss_test.go | 42 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/e2e/vmss.go b/e2e/vmss.go index 786bf7799ae..ae40bff563c 100644 --- a/e2e/vmss.go +++ b/e2e/vmss.go @@ -1197,10 +1197,7 @@ func injectWriteFilesEntriesToBoothookCustomData(decoded []byte, entries []Custo return "", err } - insertPos := strings.Index(boothookStr, "/bin/bash /opt/azure/containers/aks-node-controller-launcher.sh") - if insertPos == -1 { - insertPos = strings.Index(boothookStr, "systemctl start --no-block aks-node-controller.service") - } + insertPos := boothookInsertionPoint(boothookStr) if insertPos == -1 { return "", fmt.Errorf("cloud-boothook customData missing aks-node-controller service start") } @@ -1209,6 +1206,21 @@ func injectWriteFilesEntriesToBoothookCustomData(decoded []byte, entries []Custo return base64.StdEncoding.EncodeToString([]byte(boothookStr)), nil } +func boothookInsertionPoint(boothookStr string) int { + launcherPos := strings.Index(boothookStr, "/opt/azure/containers/aks-node-controller-launcher.sh") + if launcherPos == -1 { + launcherPos = strings.Index(boothookStr, "systemctl start --no-block aks-node-controller.service") + } + if launcherPos == -1 { + return -1 + } + lineStart := strings.LastIndex(boothookStr[:launcherPos], "\n") + if lineStart == -1 { + return 0 + } + return lineStart + 1 +} + func renderBoothookWriteFilesEntries(entries []CustomDataWriteFile) (string, error) { var entryBuilder strings.Builder entryBuilder.WriteString("\n") diff --git a/e2e/vmss_test.go b/e2e/vmss_test.go index efc7aa7a11d..dbecf3600c1 100644 --- a/e2e/vmss_test.go +++ b/e2e/vmss_test.go @@ -1,6 +1,8 @@ package e2e import ( + "encoding/base64" + "strings" "testing" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" @@ -90,3 +92,43 @@ func TestParseLinuxCSEMessageOutboundExitCode(t *testing.T) { }) } } + +func TestBoothookInsertionPoint(t *testing.T) { + tests := []struct { + name string + boothook string + wantLine string + }{ + { + name: "nohup launcher", + boothook: "#!/bin/bash\necho before\nnohup /bin/bash /opt/azure/containers/aks-node-controller-launcher.sh >/var/log/azure/aks-node-controller.output 2>&1 &\necho after\n", + wantLine: "nohup /bin/bash /opt/azure/containers/aks-node-controller-launcher.sh", + }, + { + name: "systemd launcher fallback", + boothook: "#!/bin/bash\necho before\nsystemctl start --no-block aks-node-controller.service\necho after\n", + wantLine: "systemctl start --no-block aks-node-controller.service", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pos := boothookInsertionPoint(tt.boothook) + require.GreaterOrEqual(t, pos, 0) + require.True(t, strings.HasPrefix(tt.boothook[pos:], tt.wantLine), "insertion point should be before launcher line") + }) + } +} + +func TestInjectWriteFilesEntriesToBoothookCustomDataPreservesNohupCommand(t *testing.T) { + boothook := []byte("#!/bin/bash\necho before\nnohup /bin/bash /opt/azure/containers/aks-node-controller-launcher.sh >/var/log/azure/aks-node-controller.output 2>&1 &\necho after\n") + encoded, err := injectWriteFilesEntriesToBoothookCustomData(boothook, []CustomDataWriteFile{{ + Path: "/tmp/e2e-marker", + Content: "hello", + }}) + require.NoError(t, err) + decoded, err := base64.StdEncoding.DecodeString(encoded) + require.NoError(t, err) + got := string(decoded) + require.Contains(t, got, "\nnohup /bin/bash /opt/azure/containers/aks-node-controller-launcher.sh") + require.NotContains(t, got, "nohup\n") +} From 0deaa142e2871f0a86200850a1a87b0227e69e13 Mon Sep 17 00:00:00 2001 From: Saewon Kwak <23280628+saewoni@users.noreply.github.com> Date: Fri, 21 Aug 2026 22:27:11 +0000 Subject: [PATCH 31/38] Run local aks-node-controller for LocalDNS LPS E2E parity The LocalDNS LPS bootstrap E2E uses CustomDataWriteFiles, which disabled scriptless local-binary compilation and forced the VM to run the VHD baked-in aks-node-controller. That stale parser predates the Corefile reload change, so its provision-config Corefile differed from the baker-generated nbc-cmd Corefile, tripping compareEnvs/ValidateScriptlessPhase3. Add an opt-in ForceScriptlessCompilation scenario flag so the VM runs the locally-built aks-node-controller, making the parser and baker Corefile output consistent. Enable it for Test_LocalDNSLPSBootstrapPatch. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- e2e/scenario_localdns_hosts_test.go | 3 +++ e2e/types.go | 6 ++++++ e2e/vmss.go | 8 +++++++- 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/e2e/scenario_localdns_hosts_test.go b/e2e/scenario_localdns_hosts_test.go index 32b598c7df6..60efc30ac13 100644 --- a/e2e/scenario_localdns_hosts_test.go +++ b/e2e/scenario_localdns_hosts_test.go @@ -90,6 +90,9 @@ func Test_LocalDNSLPSBootstrapPatch(t *testing.T) { Config: Config{ Cluster: ClusterKubenet, VHD: config.VHDUbuntu2404Gen2Containerd, + // Force compiling the local aks-node-controller so the provision-config parser matches + // the baker-generated nbc-cmd for Corefile env vars (compareEnvs parity). + ForceScriptlessCompilation: true, BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { nbc.AgentPoolProfile.LocalDNSProfile.EnableLocalDNS = true }, diff --git a/e2e/types.go b/e2e/types.go index 99a7e0ac50f..03159d694f1 100644 --- a/e2e/types.go +++ b/e2e/types.go @@ -208,6 +208,12 @@ type Config struct { // CustomDataWriteFiles injects additional cloud-init write_files entries into rendered customData. // This is for e2e-only validation scenarios. CustomDataWriteFiles []CustomDataWriteFile + // ForceScriptlessCompilation forces compiling and injecting the locally-built aks-node-controller + // binary even when CustomDataWriteFiles is set. By default, scenarios that use CustomDataWriteFiles + // fall back to the VHD's baked-in aks-node-controller, whose parser may predate local changes and + // cause spurious provision-config vs nbc-cmd env diffs (e.g. Corefile template changes). Set this + // when the scenario relies on local parser/baker changes being consistent. + ForceScriptlessCompilation bool // Validator is a function where the scenario can perform any extra validation checks Validator func(ctx context.Context, s *Scenario) diff --git a/e2e/vmss.go b/e2e/vmss.go index ae40bff563c..dc70b559ec5 100644 --- a/e2e/vmss.go +++ b/e2e/vmss.go @@ -312,7 +312,13 @@ func usesScriptlessNBCCSECmd(s *Scenario) bool { } func enableScriptlessCompilation(s *Scenario) bool { - return usesScriptlessNBCCSECmd(s) && len(s.Config.CustomDataWriteFiles) <= 0 && !config.Config.DisableScriptLessCompilation && !s.Tags.NetworkIsolated && !s.VHD.Flatcar + if !usesScriptlessNBCCSECmd(s) || config.Config.DisableScriptLessCompilation || s.Tags.NetworkIsolated || s.VHD.Flatcar { + return false + } + if s.Config.ForceScriptlessCompilation { + return true + } + return len(s.Config.CustomDataWriteFiles) <= 0 } func CreateVMSSWithRetry(ctx context.Context, s *Scenario) (*ScenarioVM, error) { From 6b55b6171b51457dbf71c24834e4d16c8beb26d6 Mon Sep 17 00:00:00 2001 From: Saewon Kwak <23280628+saewoni@users.noreply.github.com> Date: Sat, 22 Aug 2026 16:26:00 +0000 Subject: [PATCH 32/38] Fix LPS E2E drop-in to overwrite localdns.sh at service path localdns.service runs /opt/azure/containers/localdns/localdns.sh, but the E2E drop-in regenerated the branch script at /opt/azure/containers/localdns.sh (wrong directory). The service therefore ran the VHD baked-in localdns.sh, which never invoked the LPS fetcher, so the fetcher stamp was never created. Write the branch script to the correct localdns/ subdirectory path. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- e2e/scenario_localdns_hosts_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/scenario_localdns_hosts_test.go b/e2e/scenario_localdns_hosts_test.go index 60efc30ac13..494d96394df 100644 --- a/e2e/scenario_localdns_hosts_test.go +++ b/e2e/scenario_localdns_hosts_test.go @@ -150,7 +150,7 @@ func mustReadCompressedLocalDNSArtifact(t *testing.T) string { func localDNSBranchScriptDropIn() string { return `[Service] -ExecStartPre=/bin/bash -c 'base64 -d ` + localDNSBranchScriptArchivePath + ` | gzip -d > /opt/azure/containers/localdns.sh && chmod 0544 /opt/azure/containers/localdns.sh' +ExecStartPre=/bin/bash -c 'base64 -d ` + localDNSBranchScriptArchivePath + ` | gzip -d > /opt/azure/containers/localdns/localdns.sh && chmod 0544 /opt/azure/containers/localdns/localdns.sh' ` } From d1495d69777b038acaf819f53472e3aad3ceb144 Mon Sep 17 00:00:00 2001 From: Saewon Kwak <23280628+saewoni@users.noreply.github.com> Date: Sat, 22 Aug 2026 16:33:36 +0000 Subject: [PATCH 33/38] Use locally-compiled aks-node-controller in LPS E2E fetcher wrapper The fake fetcher wrapper execd /opt/azure/containers/aks-node-controller (the VHD baked-in binary) for apply-localdns-config. That older binary predates the --config-file flag, so apply failed with "flag provided but not defined: -config-file" and the LPS fetch fell open, leaving the corefile .version file unwritten. Prefer the locally-compiled aks-node-controller-hotfix binary (provided by ForceScriptlessCompilation) so apply-localdns-config matches the branch code. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- e2e/scenario_localdns_hosts_test.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/e2e/scenario_localdns_hosts_test.go b/e2e/scenario_localdns_hosts_test.go index 494d96394df..a8a76440b7b 100644 --- a/e2e/scenario_localdns_hosts_test.go +++ b/e2e/scenario_localdns_hosts_test.go @@ -194,8 +194,15 @@ func localDNSLPSPayload(version string) string { func localDNSLPSFetcherWrapper() string { return `#!/bin/bash set -euo pipefail +# Prefer the locally-compiled aks-node-controller (delivered via the hotfix path by +# ForceScriptlessCompilation) so apply-localdns-config matches the branch under test; +# fall back to the VHD baked-in binary otherwise. +ANC_BIN=/opt/azure/containers/aks-node-controller +if [ -x /opt/azure/containers/aks-node-controller-hotfix ]; then + ANC_BIN=/opt/azure/containers/aks-node-controller-hotfix +fi if [ "${1:-}" != "fetch-localdns-config" ]; then - exec /opt/azure/containers/aks-node-controller "$@" + exec "$ANC_BIN" "$@" fi output="" while [ "$#" -gt 0 ]; do @@ -214,7 +221,7 @@ if [ -z "$output" ]; then exit 1 fi touch ` + localDNSFetcherStamp + ` -exec /opt/azure/containers/aks-node-controller apply-localdns-config --config-file ` + localDNSPayloadPath + ` --output "$output" +exec "$ANC_BIN" apply-localdns-config --config-file ` + localDNSPayloadPath + ` --output "$output" ` } From 089dd0cc8d1633be918037169e5734fc7d7b8d69 Mon Sep 17 00:00:00 2001 From: Saewon Kwak <23280628+saewoni@users.noreply.github.com> Date: Sat, 22 Aug 2026 16:45:54 +0000 Subject: [PATCH 34/38] Enable bootstrap live-patch annotation in LocalDNS LPS E2E localdns.sh only writes the live-patching-status node annotation when LOCALDNS_ENABLE_LEGACY_LIVEPATCH_STATUS=true; otherwise the generic knead live-patching loop owns it. knead does not drive the localDNS component in this E2E, so the annotation was never written and the validator poll timed out. Set the env var via the localdns.service drop-in so the bootstrap writer path runs and stamps components.localDNS.current. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- e2e/scenario_localdns_hosts_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/e2e/scenario_localdns_hosts_test.go b/e2e/scenario_localdns_hosts_test.go index a8a76440b7b..e62b9d6750f 100644 --- a/e2e/scenario_localdns_hosts_test.go +++ b/e2e/scenario_localdns_hosts_test.go @@ -149,7 +149,12 @@ func mustReadCompressedLocalDNSArtifact(t *testing.T) string { } func localDNSBranchScriptDropIn() string { + // LOCALDNS_ENABLE_LEGACY_LIVEPATCH_STATUS makes localdns.sh write the + // live-patching-status node annotation itself. In production the knead + // live-patching loop owns this annotation, but knead does not drive the + // localDNS component in this E2E, so opt into the bootstrap writer here. return `[Service] +Environment="LOCALDNS_ENABLE_LEGACY_LIVEPATCH_STATUS=true" ExecStartPre=/bin/bash -c 'base64 -d ` + localDNSBranchScriptArchivePath + ` | gzip -d > /opt/azure/containers/localdns/localdns.sh && chmod 0544 /opt/azure/containers/localdns/localdns.sh' ` } From 230a5262af6dafedf1778beaf96d94cec0c290ae Mon Sep 17 00:00:00 2001 From: Saewon Kwak <23280628+saewoni@users.noreply.github.com> Date: Tue, 25 Aug 2026 00:13:47 +0000 Subject: [PATCH 35/38] fix: simplify localDNS apply outcome printf in ubuntu-snapshot-update --- .../cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh b/parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh index a23e4c80b37..ed841dbabfd 100755 --- a/parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh +++ b/parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh @@ -304,11 +304,9 @@ updateLocalDNS() { echo "localDNS config apply failed" return 1 fi - printf '%s -' "${outcome}" + printf '%s\n' "${outcome}" - case "$(printf '%s -' "${outcome}" | tail -n 1)" in + case "$(printf '%s\n' "${outcome}" | tail -n 1)" in applied) if ! systemctl restart localdns.service; then echo "failed to restart localdns.service" From f5be611e2250bdd2ef27f3fc61ea148b9cc71753 Mon Sep 17 00:00:00 2001 From: Saewon Kwak <23280628+saewoni@users.noreply.github.com> Date: Tue, 25 Aug 2026 02:58:52 +0000 Subject: [PATCH 36/38] fix: fail localDNS knead apply on unexpected notFound outcome --- .../cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh b/parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh index ed841dbabfd..b631a342ab3 100755 --- a/parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh +++ b/parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh @@ -318,7 +318,10 @@ updateLocalDNS() { echo "localDNS is already current" ;; notFound) + # In the runtime apply path the payload is supplied inline, so notFound is not expected; + # treat it as a failure so knead does not checkpoint an unapplied config as succeeded. echo "localDNS LPS config is not available" + return 1 ;; noCorefileData) echo "localDNS LPS config has no node-applicable payload" From 604bb6d2efa3a5a12b4473a19d59fe82542a7316 Mon Sep 17 00:00:00 2001 From: Saewon Kwak <23280628+saewoni@users.noreply.github.com> Date: Tue, 25 Aug 2026 18:12:22 +0000 Subject: [PATCH 37/38] test: add E2E for LocalDNS LPS-unavailable fallback to baked Corefile --- e2e/scenario_localdns_hosts_test.go | 102 ++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/e2e/scenario_localdns_hosts_test.go b/e2e/scenario_localdns_hosts_test.go index e62b9d6750f..8db8dab1bf7 100644 --- a/e2e/scenario_localdns_hosts_test.go +++ b/e2e/scenario_localdns_hosts_test.go @@ -25,6 +25,7 @@ const ( localDNSFetcherStamp = "/opt/azure/containers/localdns/e2e-localdns-lps-fetcher-called" localDNSBranchScriptArchivePath = "/opt/azure/containers/localdns/e2e-localdns.sh.gz.b64" localDNSFetcherPath = "/opt/azure/containers/localdns/e2e-fetch-localdns-config" + localDNSUnavailableFetcherStamp = "/opt/azure/containers/localdns/e2e-localdns-lps-unavailable-called" ) // Test_LocalDNSHostsPlugin tests the localdns hosts plugin across all supported distros @@ -130,6 +131,58 @@ func Test_LocalDNSLPSBootstrapPatch(t *testing.T) { }) } +// Test_LocalDNSLPSUnavailableFallback validates the unhappy bootstrap path: when LPS +// has no LocalDNS config published for the node (fetch-localdns-config fails open with +// no livepatched Corefile written), localdns.sh must fall back to the baked/CSE-generated +// localdns.corefile and CoreDNS must still come up and serve DNS. This is the failure-mode +// counterpart to Test_LocalDNSLPSBootstrapPatch, exercised end-to-end on a live node. +// +// The test verifies that: +// 1. the fetcher was invoked (LPS was consulted), +// 2. no livepatched Corefile or version file is written, +// 3. updated.localdns.corefile is still produced (from the baked source), +// 4. localdns.service is enabled and resolves DNS, and +// 5. the node does NOT report a LocalDNS live-patching current version. +func Test_LocalDNSLPSUnavailableFallback(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests LocalDNS bootstrap falls back to the baked Corefile when LPS has no config", + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2404Gen2Containerd, + // Force compiling the local aks-node-controller so localdns.sh under test matches the + // branch's Corefile generation (compareEnvs parity), same as the happy-path scenario. + ForceScriptlessCompilation: true, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.AgentPoolProfile.LocalDNSProfile.EnableLocalDNS = true + }, + CustomDataWriteFiles: []CustomDataWriteFile{ + { + Path: localDNSBranchScriptArchivePath, + Permissions: "0644", + Owner: "root", + Content: mustReadCompressedLocalDNSArtifact(t), + }, + { + Path: "/etc/systemd/system/localdns.service.d/00-e2e-branch-localdns.conf", + Permissions: "0644", + Owner: "root", + Content: localDNSBranchScriptDropIn(), + }, + { + Path: localDNSFetcherPath, + Permissions: "0755", + Owner: "root", + Content: localDNSLPSUnavailableFetcherWrapper(), + }, + }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) { + config.LocalDnsProfile.EnableLocalDns = true + }, + Validator: validateLocalDNSLPSUnavailableFallback, + }, + }) +} + func mustReadCompressedLocalDNSArtifact(t *testing.T) string { t.Helper() data, err := os.ReadFile("../parts/linux/cloud-init/artifacts/localdns.sh") @@ -230,6 +283,28 @@ exec "$ANC_BIN" apply-localdns-config --config-file ` + localDNSPayloadPath + ` ` } +// localDNSLPSUnavailableFetcherWrapper simulates LPS having no LocalDNS config for the node. +// It records that fetch-localdns-config was invoked, then exits 0 without writing the output +// Corefile -- the same fail-open behavior aks-node-controller exhibits when LPS returns a +// benign "unavailable" status (NotFound/PermissionDenied/Unauthenticated). Non-fetch +// subcommands are delegated to the real binary so the rest of provisioning is unaffected. +func localDNSLPSUnavailableFetcherWrapper() string { + return `#!/bin/bash +set -euo pipefail +ANC_BIN=/opt/azure/containers/aks-node-controller +if [ -x /opt/azure/containers/aks-node-controller-hotfix ]; then + ANC_BIN=/opt/azure/containers/aks-node-controller-hotfix +fi +if [ "${1:-}" != "fetch-localdns-config" ]; then + exec "$ANC_BIN" "$@" +fi +# LPS has nothing published for this node: record the call and fail open without writing a +# livepatched Corefile, so localdns.sh falls back to the baked localdns.corefile. +touch ` + localDNSUnavailableFetcherStamp + ` +exit 0 +` +} + func validateLocalDNSLPSBootstrapPatch(ctx context.Context, s *Scenario) { const ( updatedCorefile = "/opt/azure/containers/localdns/updated.localdns.corefile" @@ -253,3 +328,30 @@ func validateLocalDNSLPSBootstrapPatch(ctx context.Context, s *Scenario) { }) require.NoError(s.T, err, "node did not report LocalDNS live-patching current version %q", desiredLocalDNSVersion) } + +func validateLocalDNSLPSUnavailableFallback(ctx context.Context, s *Scenario) { + const ( + bakedCorefile = "/opt/azure/containers/localdns/localdns.corefile" + updatedCorefile = "/opt/azure/containers/localdns/updated.localdns.corefile" + livepatchedCorefile = "/opt/azure/containers/localdns/livepatched.localdns.corefile" + ) + + // The fetcher ran (LPS was consulted) but wrote nothing, so no livepatched Corefile + // or version file should exist and localdns.sh must fall back to the baked Corefile. + ValidateFileExists(ctx, s, localDNSUnavailableFetcherStamp) + ValidateFileDoesNotExist(ctx, s, livepatchedCorefile) + ValidateFileDoesNotExist(ctx, s, livepatchedCorefile+".version") + + // CoreDNS still comes up from the baked source and serves DNS. + ValidateFileExists(ctx, s, bakedCorefile) + ValidateFileHasContent(ctx, s, updatedCorefile, "health-check.localdns.local:53") + ValidateLocalDNSService(ctx, s, "enabled") + ValidateLocalDNSResolution(ctx, s, "169.254.10.10") + + // No LocalDNS live-patching version should be reported when LPS had nothing to apply. + node, err := s.Runtime.Kube.Typed.CoreV1().Nodes().Get(ctx, s.Runtime.VM.KubeName, metav1.GetOptions{}) + require.NoError(s.T, err, "failed to get node %s", s.Runtime.VM.KubeName) + status := node.Annotations["kubernetes.azure.com/live-patching-status"] + require.NotContains(s.T, status, `"localDNS"`, + "node unexpectedly reported a LocalDNS live-patching status when LPS had no config: %q", status) +} From 7a8bffd126d4552c3373b5451984721cd4956fec Mon Sep 17 00:00:00 2001 From: Saewon Kwak <23280628+saewoni@users.noreply.github.com> Date: Thu, 3 Sep 2026 02:11:38 +0000 Subject: [PATCH 38/38] feat: add LocalDNS forward health knobs --- .../parser/templates/localdns.toml.gtpl | 13 +- .../aksnodeconfig/v1/localdns_config.pb.go | 159 +++++++++++++++--- .../aksnodeconfig/v1/localdns_config.proto | 12 ++ pkg/agent/baker.go | 12 ++ pkg/agent/datamodel/types.go | 47 +++++- 5 files changed, 206 insertions(+), 37 deletions(-) diff --git a/aks-node-controller/parser/templates/localdns.toml.gtpl b/aks-node-controller/parser/templates/localdns.toml.gtpl index 191cee3cbcb..88ee14118ca 100644 --- a/aks-node-controller/parser/templates/localdns.toml.gtpl +++ b/aks-node-controller/parser/templates/localdns.toml.gtpl @@ -45,6 +45,12 @@ health-check.localdns.local:53 { {{- end}} policy {{$forwardPolicy}} max_concurrent {{$override.MaxConcurrent}} + {{- if and $override.HealthCheck $override.HealthCheck.GetDuration}} + health_check {{$override.HealthCheck.GetDuration}}{{if $override.HealthCheck.GetNoRec}} no_rec{{end}}{{if $override.HealthCheck.GetDomain}} domain {{$override.HealthCheck.GetDomain}}{{end}} + {{- end}} + {{- if $override.GetFailfastAllUnhealthyUpstreams}} + failfast_all_unhealthy_upstreams + {{- end}} } ready {{getLocalDnsNodeListenerIp}}:8181 reload @@ -79,7 +85,6 @@ health-check.localdns.local:53 { {{- range $domain, $override := $.Config.LocalDnsProfile.KubeDnsOverrides}} {{- $isRootDomain := eq $domain "." -}} {{- $fwdToClusterCoreDNS := or (hasSuffix $domain "cluster.local") (eq $override.ForwardDestination "ClusterCoreDNS")}} -{{- $forwardPolicy := "" }} {{- $forwardPolicy := "sequential" -}} {{- if eq $override.ForwardPolicy "RoundRobin" -}} {{- $forwardPolicy = "round_robin" -}} @@ -111,6 +116,12 @@ health-check.localdns.local:53 { {{- end}} policy {{$forwardPolicy}} max_concurrent {{$override.MaxConcurrent}} + {{- if and $override.HealthCheck $override.HealthCheck.GetDuration}} + health_check {{$override.HealthCheck.GetDuration}}{{if $override.HealthCheck.GetNoRec}} no_rec{{end}}{{if $override.HealthCheck.GetDomain}} domain {{$override.HealthCheck.GetDomain}}{{end}} + {{- end}} + {{- if $override.GetFailfastAllUnhealthyUpstreams}} + failfast_all_unhealthy_upstreams + {{- end}} } ready {{getLocalDnsClusterListenerIp}}:8181 reload diff --git a/aks-node-controller/pkg/gen/aksnodeconfig/v1/localdns_config.pb.go b/aks-node-controller/pkg/gen/aksnodeconfig/v1/localdns_config.pb.go index d3981903347..95c2bb724d4 100644 --- a/aks-node-controller/pkg/gen/aksnodeconfig/v1/localdns_config.pb.go +++ b/aks-node-controller/pkg/gen/aksnodeconfig/v1/localdns_config.pb.go @@ -159,6 +159,10 @@ type LocalDnsOverrides struct { ServeStaleDurationInSeconds *int32 `protobuf:"varint,7,opt,name=serve_stale_duration_in_seconds,json=serveStaleDurationInSeconds,proto3,oneof" json:"serve_stale_duration_in_seconds,omitempty"` // Policy for serving stale data. ServeStale string `protobuf:"bytes,8,opt,name=serve_stale,json=serveStale,proto3" json:"serve_stale,omitempty"` + // Fail requests when all configured upstreams are unhealthy. + FailfastAllUnhealthyUpstreams *bool `protobuf:"varint,9,opt,name=failfast_all_unhealthy_upstreams,json=failfastAllUnhealthyUpstreams,proto3,oneof" json:"failfast_all_unhealthy_upstreams,omitempty"` + // Interval used by the forward plugin for upstream health checks, for example "500ms". + HealthCheck *LocalDnsHealthCheck `protobuf:"bytes,10,opt,name=health_check,json=healthCheck,proto3" json:"health_check,omitempty"` } func (x *LocalDnsOverrides) Reset() { @@ -247,6 +251,81 @@ func (x *LocalDnsOverrides) GetServeStale() string { return "" } +func (x *LocalDnsOverrides) GetFailfastAllUnhealthyUpstreams() bool { + if x != nil && x.FailfastAllUnhealthyUpstreams != nil { + return *x.FailfastAllUnhealthyUpstreams + } + return false +} + +func (x *LocalDnsOverrides) GetHealthCheck() *LocalDnsHealthCheck { + if x != nil { + return x.HealthCheck + } + return nil +} + +type LocalDnsHealthCheck struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Duration *string `protobuf:"bytes,1,opt,name=duration,proto3,oneof" json:"duration,omitempty"` + NoRec *bool `protobuf:"varint,2,opt,name=no_rec,json=noRec,proto3,oneof" json:"no_rec,omitempty"` + Domain *string `protobuf:"bytes,3,opt,name=domain,proto3,oneof" json:"domain,omitempty"` +} + +func (x *LocalDnsHealthCheck) Reset() { + *x = LocalDnsHealthCheck{} + mi := &file_aksnodeconfig_v1_localdns_config_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LocalDnsHealthCheck) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LocalDnsHealthCheck) ProtoMessage() {} + +func (x *LocalDnsHealthCheck) ProtoReflect() protoreflect.Message { + mi := &file_aksnodeconfig_v1_localdns_config_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LocalDnsHealthCheck.ProtoReflect.Descriptor instead. +func (*LocalDnsHealthCheck) Descriptor() ([]byte, []int) { + return file_aksnodeconfig_v1_localdns_config_proto_rawDescGZIP(), []int{2} +} + +func (x *LocalDnsHealthCheck) GetDuration() string { + if x != nil && x.Duration != nil { + return *x.Duration + } + return "" +} + +func (x *LocalDnsHealthCheck) GetNoRec() bool { + if x != nil && x.NoRec != nil { + return *x.NoRec + } + return false +} + +func (x *LocalDnsHealthCheck) GetDomain() string { + if x != nil && x.Domain != nil { + return *x.Domain + } + return "" +} + var File_aksnodeconfig_v1_localdns_config_proto protoreflect.FileDescriptor var file_aksnodeconfig_v1_localdns_config_proto_rawDesc = []byte{ @@ -307,7 +386,7 @@ var file_aksnodeconfig_v1_localdns_config_proto_rawDesc = []byte{ 0x6d, 0x6f, 0x72, 0x79, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x5f, 0x69, 0x6e, 0x5f, 0x6d, 0x62, 0x42, 0x2b, 0x0a, 0x29, 0x5f, 0x68, 0x6f, 0x73, 0x74, 0x73, 0x5f, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x5f, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, - 0x61, 0x6c, 0x5f, 0x69, 0x6e, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x22, 0xd9, 0x03, + 0x61, 0x6c, 0x5f, 0x69, 0x6e, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x22, 0x96, 0x05, 0x0a, 0x11, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x44, 0x6e, 0x73, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x71, 0x75, 0x65, 0x72, 0x79, 0x5f, 0x6c, 0x6f, 0x67, 0x67, 0x69, 0x6e, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x71, 0x75, 0x65, 0x72, @@ -332,18 +411,39 @@ var file_aksnodeconfig_v1_localdns_config_proto_rawDesc = []byte{ 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x88, 0x01, 0x01, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x65, 0x5f, 0x73, 0x74, 0x61, 0x6c, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x65, 0x72, 0x76, 0x65, 0x53, - 0x74, 0x61, 0x6c, 0x65, 0x42, 0x11, 0x0a, 0x0f, 0x5f, 0x6d, 0x61, 0x78, 0x5f, 0x63, 0x6f, 0x6e, - 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x42, 0x1c, 0x0a, 0x1a, 0x5f, 0x63, 0x61, 0x63, 0x68, - 0x65, 0x5f, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x6e, 0x5f, 0x73, 0x65, - 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x42, 0x22, 0x0a, 0x20, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x5f, - 0x73, 0x74, 0x61, 0x6c, 0x65, 0x5f, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, - 0x6e, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x42, 0x5a, 0x5a, 0x58, 0x67, 0x69, 0x74, - 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x2f, 0x61, 0x67, - 0x65, 0x6e, 0x74, 0x62, 0x61, 0x6b, 0x65, 0x72, 0x2f, 0x61, 0x6b, 0x73, 0x2d, 0x6e, 0x6f, 0x64, - 0x65, 0x2d, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2f, 0x70, 0x6b, 0x67, - 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x61, 0x6b, 0x73, 0x6e, 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x2f, 0x76, 0x31, 0x3b, 0x61, 0x6b, 0x73, 0x6e, 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x74, 0x61, 0x6c, 0x65, 0x12, 0x4c, 0x0a, 0x20, 0x66, 0x61, 0x69, 0x6c, 0x66, 0x61, 0x73, 0x74, + 0x5f, 0x61, 0x6c, 0x6c, 0x5f, 0x75, 0x6e, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x79, 0x5f, 0x75, + 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x48, 0x03, + 0x52, 0x1d, 0x66, 0x61, 0x69, 0x6c, 0x66, 0x61, 0x73, 0x74, 0x41, 0x6c, 0x6c, 0x55, 0x6e, 0x68, + 0x65, 0x61, 0x6c, 0x74, 0x68, 0x79, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x73, 0x88, + 0x01, 0x01, 0x12, 0x48, 0x0a, 0x0c, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x5f, 0x63, 0x68, 0x65, + 0x63, 0x6b, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x61, 0x6b, 0x73, 0x6e, 0x6f, + 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x6f, 0x63, 0x61, + 0x6c, 0x44, 0x6e, 0x73, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, + 0x0b, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x42, 0x11, 0x0a, 0x0f, + 0x5f, 0x6d, 0x61, 0x78, 0x5f, 0x63, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x42, + 0x1c, 0x0a, 0x1a, 0x5f, 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x5f, 0x69, 0x6e, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x42, 0x22, 0x0a, + 0x20, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x5f, 0x73, 0x74, 0x61, 0x6c, 0x65, 0x5f, 0x64, 0x75, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x6e, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, + 0x73, 0x42, 0x23, 0x0a, 0x21, 0x5f, 0x66, 0x61, 0x69, 0x6c, 0x66, 0x61, 0x73, 0x74, 0x5f, 0x61, + 0x6c, 0x6c, 0x5f, 0x75, 0x6e, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x79, 0x5f, 0x75, 0x70, 0x73, + 0x74, 0x72, 0x65, 0x61, 0x6d, 0x73, 0x22, 0x92, 0x01, 0x0a, 0x13, 0x4c, 0x6f, 0x63, 0x61, 0x6c, + 0x44, 0x6e, 0x73, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x1f, + 0x0a, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x48, 0x00, 0x52, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x88, 0x01, 0x01, 0x12, + 0x1a, 0x0a, 0x06, 0x6e, 0x6f, 0x5f, 0x72, 0x65, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x48, + 0x01, 0x52, 0x05, 0x6e, 0x6f, 0x52, 0x65, 0x63, 0x88, 0x01, 0x01, 0x12, 0x1b, 0x0a, 0x06, 0x64, + 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x02, 0x52, 0x06, 0x64, + 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x88, 0x01, 0x01, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x64, 0x75, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x6e, 0x6f, 0x5f, 0x72, 0x65, 0x63, + 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x42, 0x5a, 0x5a, 0x58, 0x67, + 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x2f, + 0x61, 0x67, 0x65, 0x6e, 0x74, 0x62, 0x61, 0x6b, 0x65, 0x72, 0x2f, 0x61, 0x6b, 0x73, 0x2d, 0x6e, + 0x6f, 0x64, 0x65, 0x2d, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2f, 0x70, + 0x6b, 0x67, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x61, 0x6b, 0x73, 0x6e, 0x6f, 0x64, 0x65, 0x63, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x2f, 0x76, 0x31, 0x3b, 0x61, 0x6b, 0x73, 0x6e, 0x6f, 0x64, 0x65, 0x63, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -358,23 +458,25 @@ func file_aksnodeconfig_v1_localdns_config_proto_rawDescGZIP() []byte { return file_aksnodeconfig_v1_localdns_config_proto_rawDescData } -var file_aksnodeconfig_v1_localdns_config_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_aksnodeconfig_v1_localdns_config_proto_msgTypes = make([]protoimpl.MessageInfo, 5) var file_aksnodeconfig_v1_localdns_config_proto_goTypes = []any{ - (*LocalDnsProfile)(nil), // 0: aksnodeconfig.v1.LocalDnsProfile - (*LocalDnsOverrides)(nil), // 1: aksnodeconfig.v1.LocalDnsOverrides - nil, // 2: aksnodeconfig.v1.LocalDnsProfile.VnetDnsOverridesEntry - nil, // 3: aksnodeconfig.v1.LocalDnsProfile.KubeDnsOverridesEntry + (*LocalDnsProfile)(nil), // 0: aksnodeconfig.v1.LocalDnsProfile + (*LocalDnsOverrides)(nil), // 1: aksnodeconfig.v1.LocalDnsOverrides + (*LocalDnsHealthCheck)(nil), // 2: aksnodeconfig.v1.LocalDnsHealthCheck + nil, // 3: aksnodeconfig.v1.LocalDnsProfile.VnetDnsOverridesEntry + nil, // 4: aksnodeconfig.v1.LocalDnsProfile.KubeDnsOverridesEntry } var file_aksnodeconfig_v1_localdns_config_proto_depIdxs = []int32{ - 2, // 0: aksnodeconfig.v1.LocalDnsProfile.vnet_dns_overrides:type_name -> aksnodeconfig.v1.LocalDnsProfile.VnetDnsOverridesEntry - 3, // 1: aksnodeconfig.v1.LocalDnsProfile.kube_dns_overrides:type_name -> aksnodeconfig.v1.LocalDnsProfile.KubeDnsOverridesEntry - 1, // 2: aksnodeconfig.v1.LocalDnsProfile.VnetDnsOverridesEntry.value:type_name -> aksnodeconfig.v1.LocalDnsOverrides - 1, // 3: aksnodeconfig.v1.LocalDnsProfile.KubeDnsOverridesEntry.value:type_name -> aksnodeconfig.v1.LocalDnsOverrides - 4, // [4:4] is the sub-list for method output_type - 4, // [4:4] is the sub-list for method input_type - 4, // [4:4] is the sub-list for extension type_name - 4, // [4:4] is the sub-list for extension extendee - 0, // [0:4] is the sub-list for field type_name + 3, // 0: aksnodeconfig.v1.LocalDnsProfile.vnet_dns_overrides:type_name -> aksnodeconfig.v1.LocalDnsProfile.VnetDnsOverridesEntry + 4, // 1: aksnodeconfig.v1.LocalDnsProfile.kube_dns_overrides:type_name -> aksnodeconfig.v1.LocalDnsProfile.KubeDnsOverridesEntry + 2, // 2: aksnodeconfig.v1.LocalDnsOverrides.health_check:type_name -> aksnodeconfig.v1.LocalDnsHealthCheck + 1, // 3: aksnodeconfig.v1.LocalDnsProfile.VnetDnsOverridesEntry.value:type_name -> aksnodeconfig.v1.LocalDnsOverrides + 1, // 4: aksnodeconfig.v1.LocalDnsProfile.KubeDnsOverridesEntry.value:type_name -> aksnodeconfig.v1.LocalDnsOverrides + 5, // [5:5] is the sub-list for method output_type + 5, // [5:5] is the sub-list for method input_type + 5, // [5:5] is the sub-list for extension type_name + 5, // [5:5] is the sub-list for extension extendee + 0, // [0:5] is the sub-list for field type_name } func init() { file_aksnodeconfig_v1_localdns_config_proto_init() } @@ -384,13 +486,14 @@ func file_aksnodeconfig_v1_localdns_config_proto_init() { } file_aksnodeconfig_v1_localdns_config_proto_msgTypes[0].OneofWrappers = []any{} file_aksnodeconfig_v1_localdns_config_proto_msgTypes[1].OneofWrappers = []any{} + file_aksnodeconfig_v1_localdns_config_proto_msgTypes[2].OneofWrappers = []any{} type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_aksnodeconfig_v1_localdns_config_proto_rawDesc, NumEnums: 0, - NumMessages: 4, + NumMessages: 5, NumExtensions: 0, NumServices: 0, }, diff --git a/aks-node-controller/proto/aksnodeconfig/v1/localdns_config.proto b/aks-node-controller/proto/aksnodeconfig/v1/localdns_config.proto index 8f11b898ee9..36e38e42785 100644 --- a/aks-node-controller/proto/aksnodeconfig/v1/localdns_config.proto +++ b/aks-node-controller/proto/aksnodeconfig/v1/localdns_config.proto @@ -62,4 +62,16 @@ message LocalDnsOverrides { // Policy for serving stale data. string serve_stale = 8; + + // Fail requests when all configured upstreams are unhealthy. + optional bool failfast_all_unhealthy_upstreams = 9; + + // Interval used by the forward plugin for upstream health checks, for example "500ms". + LocalDnsHealthCheck health_check = 10; +} + +message LocalDnsHealthCheck { + optional string duration = 1; + optional bool no_rec = 2; + optional string domain = 3; } diff --git a/pkg/agent/baker.go b/pkg/agent/baker.go index f0801a0b70a..0f7899ebe60 100644 --- a/pkg/agent/baker.go +++ b/pkg/agent/baker.go @@ -2308,6 +2308,12 @@ health-check.localdns.local:53 { {{- end}} policy {{$forwardPolicy}} max_concurrent {{$override.MaxConcurrent}} + {{- if and $override.HealthCheck $override.HealthCheck.GetDuration}} + health_check {{$override.HealthCheck.GetDuration}}{{if $override.HealthCheck.GetNoRec}} no_rec{{end}}{{if $override.HealthCheck.GetDomain}} domain {{$override.HealthCheck.GetDomain}}{{end}} + {{- end}} + {{- if $override.GetFailfastAllUnhealthyUpstreams}} + failfast_all_unhealthy_upstreams + {{- end}} } ready {{$.NodeListenerIP}}:8181 reload @@ -2374,6 +2380,12 @@ health-check.localdns.local:53 { {{- end}} policy {{$forwardPolicy}} max_concurrent {{$override.MaxConcurrent}} + {{- if and $override.HealthCheck $override.HealthCheck.GetDuration}} + health_check {{$override.HealthCheck.GetDuration}}{{if $override.HealthCheck.GetNoRec}} no_rec{{end}}{{if $override.HealthCheck.GetDomain}} domain {{$override.HealthCheck.GetDomain}}{{end}} + {{- end}} + {{- if $override.GetFailfastAllUnhealthyUpstreams}} + failfast_all_unhealthy_upstreams + {{- end}} } ready {{$.ClusterListenerIP}}:8181 reload diff --git a/pkg/agent/datamodel/types.go b/pkg/agent/datamodel/types.go index a0d405bdb33..2f0b774ee65 100644 --- a/pkg/agent/datamodel/types.go +++ b/pkg/agent/datamodel/types.go @@ -2596,18 +2596,49 @@ type LocalDNSCoreFileData struct { IncludeHostsPlugin bool } +// LocalDNSHealthCheck represents CoreDNS forward plugin health check settings. +type LocalDNSHealthCheck struct { + Duration *string `json:"duration,omitempty"` + NoRec *bool `json:"noRec,omitempty"` + Domain *string `json:"domain,omitempty"` +} + +func (h *LocalDNSHealthCheck) GetDuration() string { + if h != nil && h.Duration != nil { + return *h.Duration + } + return "" +} + +func (h *LocalDNSHealthCheck) GetNoRec() bool { + return h != nil && h.NoRec != nil && *h.NoRec +} + +func (h *LocalDNSHealthCheck) GetDomain() string { + if h != nil && h.Domain != nil { + return *h.Domain + } + return "" +} + // LocalDNSOverrides represents DNS override settings for both VnetDNS and KubeDNS traffic. // VnetDNS overrides apply to DNS traffic from pods with dnsPolicy:default or kubelet (referred to as VnetDNS traffic). // KubeDNS overrides apply to DNS traffic from pods with dnsPolicy:ClusterFirst (referred to as KubeDNS traffic). type LocalDNSOverrides struct { - QueryLogging string `json:"queryLogging,omitempty"` - Protocol string `json:"protocol,omitempty"` - ForwardDestination string `json:"forwardDestination,omitempty"` - ForwardPolicy string `json:"forwardPolicy,omitempty"` - MaxConcurrent *int32 `json:"maxConcurrent,omitempty"` - CacheDurationInSeconds *int32 `json:"cacheDurationInSeconds,omitempty"` - ServeStaleDurationInSeconds *int32 `json:"serveStaleDurationInSeconds,omitempty"` - ServeStale string `json:"serveStale,omitempty"` + QueryLogging string `json:"queryLogging,omitempty"` + Protocol string `json:"protocol,omitempty"` + ForwardDestination string `json:"forwardDestination,omitempty"` + ForwardPolicy string `json:"forwardPolicy,omitempty"` + MaxConcurrent *int32 `json:"maxConcurrent,omitempty"` + CacheDurationInSeconds *int32 `json:"cacheDurationInSeconds,omitempty"` + ServeStaleDurationInSeconds *int32 `json:"serveStaleDurationInSeconds,omitempty"` + ServeStale string `json:"serveStale,omitempty"` + FailfastAllUnhealthyUpstreams *bool `json:"failfastAllUnhealthyUpstreams,omitempty"` + HealthCheck *LocalDNSHealthCheck `json:"healthCheck,omitempty"` +} + +func (o *LocalDNSOverrides) GetFailfastAllUnhealthyUpstreams() bool { + return o != nil && o.FailfastAllUnhealthyUpstreams != nil && *o.FailfastAllUnhealthyUpstreams } // ShouldEnableLocalDNS returns true if AgentPoolProfile, LocalDNSProfile is not nil and