diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index c056cf6e093..34e8e69549d 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -17,9 +17,9 @@ Windows VHD is configured through [VHD](./vhdbuilder/packer/windows/windows-vhd- [apiserver](./apiserver/) is `go` based webserver. It receives request from external client and generates CSE and CustomData to be used on the VHD when a new node is created / provisioned. -windows generates its CSE package using [script](./parts/windows/kuberneteswindowssetup.ps1). +windows generates its CSE package using [script](./parts/windows/kuberneteswindowssetup.ps1.template). -Both CSE scripts run in two phases — Windows `BasePrep`/`NodePrep` (`kuberneteswindowssetup.ps1`) and Linux `basePrep`/`nodePrep` (`cse_main.sh`). `basePrep` is gated only by the `base_prep.complete` marker, so it is **skipped on a PIS-cached VHD** (the marker is baked in); `nodePrep` runs whenever `PreProvisionOnly` is false — every real node. So node-specific, secret, or expiring data (e.g. the TLS bootstrap token / kubeconfig) must be written in `nodePrep`, not `basePrep`, or the baked copy goes stale. On PIS the phases are separate VM runs — variables re-initialize from the real node's live CustomData, so `nodePrep` can't rely on `basePrep` state. (Non-PIS: both run sequentially in one execution, no reboot between phases.) +Both CSE scripts run in two phases — Windows `BasePrep`/`NodePrep` (`kuberneteswindowssetup.ps1.template`) and Linux `basePrep`/`nodePrep` (`cse_main.sh`). `basePrep` is gated only by the `base_prep.complete` marker, so it is **skipped on a PIS-cached VHD** (the marker is baked in); `nodePrep` runs whenever `PreProvisionOnly` is false — every real node. So node-specific, secret, or expiring data (e.g. the TLS bootstrap token / kubeconfig) must be written in `nodePrep`, not `basePrep`, or the baked copy goes stale. On PIS the phases are separate VM runs — variables re-initialize from the real node's live CustomData, so `nodePrep` can't rely on `basePrep` state. (Non-PIS: both run sequentially in one execution, no reboot between phases.) The webserver is also used to determine the latest version of Linux VHDs available for provisioning within AKS clusters. @@ -145,7 +145,7 @@ Analyze PRs for these compatibility scenarios: - Hardcoded paths that differ between deployment modes **4. PIS / VHD Caching — basePrep vs nodePrep split (Windows + Linux)** -- **Context**: PIS bakes a VHD from a temporary VM, then boots many real nodes from it. Same model in Windows `parts/windows/kuberneteswindowssetup.ps1` (`BasePrep`/`NodePrep`) and Linux `parts/linux/cloud-init/artifacts/cse_main.sh` (`basePrep`/`nodePrep`): +- **Context**: PIS bakes a VHD from a temporary VM, then boots many real nodes from it. Same model in Windows `parts/windows/kuberneteswindowssetup.ps1.template` (`BasePrep`/`NodePrep`) and Linux `parts/linux/cloud-init/artifacts/cse_main.sh` (`basePrep`/`nodePrep`): - `basePrep` — gated only by the `base_prep.complete` marker (`C:\AzureData\` Windows, `/opt/azure/containers/` Linux). **Skipped on PIS real nodes** because the marker is baked into the VHD. - `nodePrep` — runs whenever `PreProvisionOnly` is false (every real node); skipped only on the bake VM. - The bake run sets `PreProvisionOnly=true` (`{{GetPreProvisionOnly}}`) and writes the marker after `basePrep` succeeds (Windows `finally`; Linux `cse_start.sh`). diff --git a/.github/renovate.json b/.github/renovate.json index d8f868dbdad..b958a649df2 100644 --- a/.github/renovate.json +++ b/.github/renovate.json @@ -58,7 +58,8 @@ "custom.deb2004", "custom.deb2204", "custom.deb2404", - "custom.deb2404-test" + "custom.deb2404-test", + "custom.deb2604" ], "matchUpdateTypes": [ "patch" @@ -845,6 +846,20 @@ "versioningTemplate": "deb", "autoReplaceStringTemplate": "\"renovateTag\": \"name={{{packageName}}}, repository=test, os=ubuntu, release=24.04\",\n \"latestVersion\": \"{{{newValue}}}\"{{#if depType}},\n \"previousLatestVersion\": \"{{{currentValue}}}\"{{/if}}" }, + { + "customType": "regex", + "description": "auto update packages for OS ubuntu 26.04 in components.json", + "managerFilePatterns": [ + "/parts/common/components.json/" + ], + "matchStringsStrategy": "any", + "matchStrings": [ + "\"renovateTag\":\\s*\"name=(?[^\"]+), repository=production, os=ubuntu, release=26\\.04\",\\s*\"latestVersion\":\\s*\"(?[^\"]+)\"(?:[^}]*\"previousLatestVersion\":\\s*\"(?[^\"]+)\")?" + ], + "datasourceTemplate": "custom.deb2604", + "versioningTemplate": "deb", + "autoReplaceStringTemplate": "\"renovateTag\": \"name={{{packageName}}}, repository=production, os=ubuntu, release=26.04\",\n \"latestVersion\": \"{{{newValue}}}\"{{#if depType}},\n \"previousLatestVersion\": \"{{{currentValue}}}\"{{/if}}" + }, { "customType": "regex", "description": "auto update packages for OS Mariner 2.0 in components.json", @@ -996,6 +1011,13 @@ "transformTemplates": [ "{\"releases\": $map(($index := releases#$i[version=\"Package: {{packageName}}\"].$i; $map($index, function($i) { $substringAfter(releases[$i + 1].version, \"Version: \") })), function($v) { {\"version\": $v} })[]}" ] + }, + "deb2604": { + "defaultRegistryUrlTemplate": "https://packages.microsoft.com/ubuntu/26.04/prod/dists/resolute/main/binary-amd64/Packages", + "format": "plain", + "transformTemplates": [ + "{\"releases\": $map(($index := releases#$i[version=\"Package: {{packageName}}\"].$i; $map($index, function($i) { $substringAfter(releases[$i + 1].version, \"Version: \") })), function($v) { {\"version\": $v} })[]}" + ] } } } diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index b401379da51..cffb0d14d5f 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -48,7 +48,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v4.37.3 + uses: github/codeql-action/init@v4.37.6 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -62,7 +62,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v4.37.3 + uses: github/codeql-action/autobuild@v4.37.6 # ℹ️ Command-line programs to run using the OS shell. # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun @@ -75,4 +75,4 @@ jobs: # ./location_of_script_within_repo/buildscript.sh - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v4.37.3 + uses: github/codeql-action/analyze@v4.37.6 diff --git a/.pipelines/.vsts-vhd-builder.yaml b/.pipelines/.vsts-vhd-builder.yaml index 99de84a985e..8e902d06e57 100644 --- a/.pipelines/.vsts-vhd-builder.yaml +++ b/.pipelines/.vsts-vhd-builder.yaml @@ -183,6 +183,32 @@ stages: parameters: artifactName: azurelinuxv3-gen2 + # Kata VHDs are built here so that VHD-side Kata changes (kata packages, erofs tooling, + # kata configuration files) are covered by the E2E stage below. Without this job no Kata + # image carries this build's buildId tag, so Test_AzureLinuxV3Gen2Kata silently skips + # (the E2E stage sets IgnoreScenariosWithMissingVhd: true). + - job: buildAzureLinuxV3gen2kata + timeoutInMinutes: 360 + steps: + - bash: | + echo '##vso[task.setvariable variable=OS_SKU]AzureLinux' + echo '##vso[task.setvariable variable=OS_VERSION]V3kata' + echo '##vso[task.setvariable variable=IMG_PUBLISHER]MicrosoftCBLMariner' + echo '##vso[task.setvariable variable=IMG_OFFER]azure-linux-3' + echo '##vso[task.setvariable variable=IMG_SKU]azure-linux-3-gen2' + echo '##vso[task.setvariable variable=IMG_VERSION]latest' + echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' + echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_D16ads_v5' + echo '##vso[task.setvariable variable=FEATURE_FLAGS]kata' + echo '##vso[task.setvariable variable=ARCHITECTURE]X86_64' + echo '##vso[task.setvariable variable=ENABLE_FIPS]false' + echo '##vso[task.setvariable variable=ENABLE_TRUSTED_LAUNCH]False' + echo '##vso[task.setvariable variable=ENABLE_CGROUPV2]True' + displayName: Setup Build Variables + - template: ./templates/.builder-release-template.yaml + parameters: + artifactName: azurelinuxv3-gen2-kata + - job: buildAzureLinuxV3ARM64gen2fips timeoutInMinutes: 360 steps: diff --git a/.pipelines/e2e-gpu.yaml b/.pipelines/e2e-gpu.yaml index af36f7f6920..403e6298264 100644 --- a/.pipelines/e2e-gpu.yaml +++ b/.pipelines/e2e-gpu.yaml @@ -37,5 +37,5 @@ jobs: parameters: name: Ubuntu GPU Tests IgnoreScenariosWithMissingVhd: false - jobTimeoutInMinutes: 120 + jobTimeoutInMinutes: 180 failedTestsRetryCount: ${{ variables.E2E_FAILED_TESTS_RETRY_COUNT }} diff --git a/AGENTS.md b/AGENTS.md index 93108d87d1f..13b0e949e10 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -17,9 +17,9 @@ Windows VHD is configured through [VHD](./vhdbuilder/packer/windows/windows-vhd- [apiserver](./apiserver/) is `go` based webserver. It receives request from external client and generates CSE and CustomData to be used on the VHD when a new node is created / provisioned. -windows generates its CSE package using [script](./parts/windows/kuberneteswindowssetup.ps1). +windows generates its CSE package using [script](./parts/windows/kuberneteswindowssetup.ps1.template). -Both CSE scripts run in two phases — Windows `BasePrep`/`NodePrep` (`kuberneteswindowssetup.ps1`) and Linux `basePrep`/`nodePrep` (`cse_main.sh`). `basePrep` is gated only by the `base_prep.complete` marker, so it is **skipped on a PIS-cached VHD** (the marker is baked in); `nodePrep` runs whenever `PreProvisionOnly` is false — every real node. So node-specific, secret, or expiring data (e.g. the TLS bootstrap token / kubeconfig) must be written in `nodePrep`, not `basePrep`, or the baked copy goes stale. On PIS the phases are separate VM runs — variables re-initialize from the real node's live CustomData, so `nodePrep` can't rely on `basePrep` state. (Non-PIS: both run sequentially in one execution, no reboot between phases.) +Both CSE scripts run in two phases — Windows `BasePrep`/`NodePrep` (`kuberneteswindowssetup.ps1.template`) and Linux `basePrep`/`nodePrep` (`cse_main.sh`). `basePrep` is gated only by the `base_prep.complete` marker, so it is **skipped on a PIS-cached VHD** (the marker is baked in); `nodePrep` runs whenever `PreProvisionOnly` is false — every real node. So node-specific, secret, or expiring data (e.g. the TLS bootstrap token / kubeconfig) must be written in `nodePrep`, not `basePrep`, or the baked copy goes stale. On PIS the phases are separate VM runs — variables re-initialize from the real node's live CustomData, so `nodePrep` can't rely on `basePrep` state. (Non-PIS: both run sequentially in one execution, no reboot between phases.) The webserver is also used to determine the latest version of Linux VHDs available for provisioning within AKS clusters. @@ -27,8 +27,6 @@ The webserver is also used to determine the latest version of Linux VHDs availab [parts](./parts/) serves both AgentBaker Service and VHD build. AgentBaker service and VHDs are coupled because of this shared component. When building VHD, packer maps and renames scripts from [parts](./parts/) depending on the OS / versions. The mappings can be found at [packer](./vhdbuilder/packer/). -> **IMPORTANT**: When making changes to files in the `parts` or `pkg` directories, you must run `make generate` afterward to regenerate the snapshot test data. This ensures consistency between the code and tests and prevents regressions. - Windows uses a different folder [cse](./staging/cse/windows/) for almost the same purpose. There are subtle differences as windows CSEs can be downloaded as a zip file during provisioning time due to restrictions on the file size on Windows system, while for linux based systems the cse/custom data are dropped in during provisioning time. ## Deployment and Release @@ -61,7 +59,7 @@ The operational goals of this project are: When making changes, reason whether the file is used in VHD building stage, or provision stage, or both. Make sure the changes are valid in its life stage. as an example, [windows-vhd-configuration.ps1](./vhdbuilder/packer/windows/windows-vhd-configuration.ps1) defines container images to be cached in VHD, while [configure-windows-vhd.ps1](./vhdbuilder/packer/windows/configure-windows-vhd.ps1) executes commands at provision time. -One way to debug / explore / just for fun is to run [e2e](./e2e/) tests. To run locally, follow the readme file under that folder. +One way to debug / explore / just for fun is to run [e2e](./e2e/) tests. To run locally, follow the readme file under that folder. The SRE guidelines ground other coding guidelines and practices. @@ -94,6 +92,7 @@ When reviewing pull requests, perform breaking change analysis to prevent regres Analyze PRs for these compatibility scenarios: **1. Linux Provisioning Script Changes** + - **Context**: Scripts in `parts/linux/cloud-init/artifacts/` run during critical VM bootstrap and are used in both: - VHD build (uploaded via packer configs in `vhdbuilder/packer/*.json`) - VM provisioning (CSE - embedded in Go service via `pkg/agent/const.go`) @@ -118,6 +117,7 @@ Analyze PRs for these compatibility scenarios: - **ANC hotfix entry removal**: If a PR removes or modifies the `hotfix-scripts: auto-generated` block in `parts/linux/cloud-init/nodecustomdata.yml`, or resets `parts/linux/cloud-init/artifacts/aks-node-controller-hotfix.json` to `{}`, **always confirm with the PR owner** that all affected VHDs have been republished with the fix baked in or are out of the 6-month support window. Premature removal means nodes provisioned via scale-up on the old buggy VHD will no longer receive the hotfix. These files are auto-generated by `hotfix/hotfix_generate.py` (via the `hotfix-generate` GH Action) — see that script for how `version`/`scripts_version` are computed. **2. Windows Bidirectional Compatibility** + - **Context**: Windows VHD and CSE scripts release on different cadences with no guaranteed order - **What to check**: Changes to `staging/cse/windows/` (CSE scripts) or `vhdbuilder/packer/windows/` (VHD scripts) - **Breaking signals**: @@ -127,6 +127,7 @@ Analyze PRs for these compatibility scenarios: - Removing PowerShell functions or cmdlets that the other component might call **3. aks-node-controller Migration (Dual-Mode Support)** + - **Context**: Transitioning from uploading scripts during both VHD build and CSE to only uploading aks-node-controller during VHD build - **What to check**: Any changes must work in BOTH deployment modes - **Breaking signals**: @@ -136,7 +137,8 @@ Analyze PRs for these compatibility scenarios: - Hardcoded paths that differ between deployment modes **4. PIS / VHD Caching — basePrep vs nodePrep split (Windows + Linux)** -- **Context**: PIS bakes a VHD from a temporary VM, then boots many real nodes from it. Same model in Windows `parts/windows/kuberneteswindowssetup.ps1` (`BasePrep`/`NodePrep`) and Linux `parts/linux/cloud-init/artifacts/cse_main.sh` (`basePrep`/`nodePrep`): + +- **Context**: PIS bakes a VHD from a temporary VM, then boots many real nodes from it. Same model in Windows `parts/windows/kuberneteswindowssetup.ps1.template` (`BasePrep`/`NodePrep`) and Linux `parts/linux/cloud-init/artifacts/cse_main.sh` (`basePrep`/`nodePrep`): - `basePrep` — gated only by the `base_prep.complete` marker (`C:\AzureData\` Windows, `/opt/azure/containers/` Linux). **Skipped on PIS real nodes** because the marker is baked into the VHD. - `nodePrep` — runs whenever `PreProvisionOnly` is false (every real node); skipped only on the bake VM. - The bake run sets `PreProvisionOnly=true` (`{{GetPreProvisionOnly}}`) and writes the marker after `basePrep` succeeds (Windows `finally`; Linux `cse_start.sh`). @@ -150,6 +152,7 @@ Analyze PRs for these compatibility scenarios: - **Don't flag**: plain variable reads (live on the real node from CustomData); cached binaries/packages/images; the `base_prep.complete` marker; cluster-wide non-secrets (CA cert, apiserver FQDN, service CIDR); pre-existing `basePrep` writes unless the PR newly depends on them. **5. Cross-OS Compatibility** + - **What to check**: Changes work on Ubuntu, Azure Linux/Mariner, and Windows - **Breaking signals**: - Linux commands that don't work on both Ubuntu and Azure Linux/Mariner @@ -158,6 +161,7 @@ Analyze PRs for these compatibility scenarios: - Systemd differences between distributions **6. Package/Dependency Update PRs (Renovate)** + - **Context**: Renovate bot automatically creates PRs to update component versions in `parts/common/components.json`. These components are cached on VHDs during build and directly affect node stability, GPU workloads, networking, and security. Updated packages are downloaded from `packages.aks.azure.com` or upstream registries during VHD build. - **What to check**: Every version bump—even patch versions—can introduce regressions that affect production nodes. - **Analysis steps for every package update PR**: @@ -181,6 +185,7 @@ Analyze PRs for these compatibility scenarios: - **Review output for package update PRs must include a detailed version diff analysis**: **Header:** + ``` ## Package Update Analysis: **Version change**: X.Y.Z → A.B.C ( update) @@ -215,6 +220,7 @@ Analyze PRs for these compatibility scenarios: **If upstream changelog is unavailable**, explicitly state: _"Upstream changelog not found for this version range. Manual testing recommended before merge."_ **Overall risk assessment:** + ``` ### Overall Risk: 🟢 Low / 🟡 Medium / 🔴 High **Justification**: <1-2 sentence summary of why this risk level was chosen> @@ -222,6 +228,7 @@ Analyze PRs for these compatibility scenarios: ``` **Example** (for a PR like dcgm-exporter 4.7.1 → 4.8.0): + ``` ## Package Update Analysis: dcgm-exporter **Version change**: 4.7.1 → 4.8.0 (minor update) @@ -244,6 +251,7 @@ Analyze PRs for these compatibility scenarios: ### Analysis Approach **Dynamic Dependency Tracing**: + 1. For each changed file, identify what depends on it 2. Follow `source` statements in bash scripts to trace dependency chains 3. Check for function calls, variable references across files @@ -255,10 +263,12 @@ Analyze PRs for these compatibility scenarios: - Flag downloads from unauthorized sources (only packages.aks.azure.com and sources in components.json allowed) **Historical Context**: + - Look for related changes that previously caused issues - Identify patterns of fragile areas that break frequently **Test Coverage Assessment**: + - Note if changed code has e2e test coverage - Flag changes to untested areas as higher risk - Mention if new behavior lacks corresponding test additions @@ -268,6 +278,7 @@ Analyze PRs for these compatibility scenarios: Provide targeted inline comments on specific lines where you detect issues: **For each breaking change or risk:** + - Comment directly on the problematic line or code block - Explain why this is risky (e.g., "This removes function X which may be called by VHDs built in the last 6 months") - Suggest specific mitigations or alternatives @@ -296,9 +307,14 @@ Provide targeted inline comments on specific lines where you detect issues: ### Review Philosophy Think like an experienced reviewer who "eyeballs" PRs for subtle risks. Look beyond pattern matching: + - Understand the architecture and how components interact - Consider timing of releases and deployment sequences - Reason about implicit dependencies and assumptions - Flag changes that "feel risky" even without obvious red flags - Balance thoroughness with actionable feedback -- Focus on high-impact issues that could break production VM provisioning \ No newline at end of file +- Focus on high-impact issues that could break production VM provisioning +- Reason about implicit dependencies and assumptions +- Flag changes that "feel risky" even without obvious red flags +- Balance thoroughness with actionable feedback +- Focus on high-impact issues that could break production VM provisioning diff --git a/CODEOWNERS b/CODEOWNERS index 0809ee81744..fd151149cf4 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -17,7 +17,10 @@ parts/linux/cloud-init/artifacts/mariner/package-update.timer @yewmsft @YaoC parts/linux/cloud-init/artifacts/mariner/mariner-package-update.sh @yewmsft @YaoC parts/linux/cloud-init/artifacts/ubuntu/snapshot-update.service @yewmsft @YaoC parts/linux/cloud-init/artifacts/ubuntu/snapshot-update.timer @yewmsft @YaoC +parts/linux/cloud-init/artifacts/ubuntu/security-update.sh @yewmsft @YaoC parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh @yewmsft @YaoC +spec/parts/linux/cloud-init/artifacts/security-update_spec.sh @yewmsft @YaoC +spec/parts/linux/cloud-init/artifacts/snapshot-update-service_spec.sh @yewmsft @YaoC spec/parts/linux/cloud-init/artifacts/ubuntu-snapshot-update_spec.sh @yewmsft @YaoC spec/parts/linux/cloud-init/artifacts/mariner-package-update_spec.sh @yewmsft @YaoC diff --git a/aks-node-controller/app.go b/aks-node-controller/app.go index cb0e3477779..b9453fcbb83 100644 --- a/aks-node-controller/app.go +++ b/aks-node-controller/app.go @@ -66,6 +66,8 @@ type App struct { // Authorization header for the check-hotfix LPS fetch. When nil, the real IMDS endpoint // is queried. fetchAttestedToken func(ctx context.Context) (string, error) + // fetchLocalDNSConfigFn overrides the real LPS LocalDNS config fetch for tests. + fetchLocalDNSConfigFn localDNSConfigFetcher } // provision.json values are emitted as strings by the shell jq invocation. @@ -168,6 +170,33 @@ func (a *App) Run(ctx context.Context, args []string) int { return a.runCheckHotfixCommand(ctx) }, }, + { + Name: "fetch-localdns-config", + Usage: "Read the LocalDNS config from the live-patching-service and update the Corefile (fail-open)", + Flags: []cli.Flag{ + &cli.StringFlag{Name: "output", Usage: "path to write the LocalDNS Corefile"}, + }, + Action: func(ctx context.Context, cmd *cli.Command) error { + if extra := cmd.Args().Slice(); len(extra) > 0 { + slog.Warn("ignoring unexpected fetch-localdns-config arguments", "args", strings.Join(extra, " ")) + } + return a.runFetchLocalDNSConfigCommand(ctx, cmd.String("output")) + }, + }, + { + Name: "apply-localdns-config", + Usage: "Apply a dispatched LocalDNS live-patching config slice to the Corefile", + Flags: []cli.Flag{ + &cli.StringFlag{Name: "config-file", Usage: "path to the LocalDNS config JSON; reads stdin when omitted or '-'"}, + &cli.StringFlag{Name: "output", Usage: "path to write the LocalDNS Corefile"}, + }, + Action: func(ctx context.Context, cmd *cli.Command) error { + if extra := cmd.Args().Slice(); len(extra) > 0 { + return fmt.Errorf("unexpected apply-localdns-config arguments: %s", strings.Join(extra, " ")) + } + return a.runApplyLocalDNSConfigCommand(ctx, cmd.String("config-file"), cmd.String("output"), cmd.Root().Writer) + }, + }, }, } diff --git a/aks-node-controller/go.mod b/aks-node-controller/go.mod index faf133532f5..a2f9e0c202f 100644 --- a/aks-node-controller/go.mod +++ b/aks-node-controller/go.mod @@ -3,18 +3,32 @@ module github.com/Azure/agentbaker/aks-node-controller go 1.25.11 require ( + github.com/Azure/agentbaker/aks-live-patching v0.0.0 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1 github.com/Masterminds/semver/v3 v3.5.0 github.com/fsnotify/fsnotify v1.8.0 github.com/google/go-cmp v0.7.0 github.com/stretchr/testify v1.11.1 github.com/urfave/cli/v3 v3.8.0 - google.golang.org/protobuf v1.36.7 + google.golang.org/grpc v1.82.1 + google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 ) require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/kr/pretty v0.3.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect + golang.org/x/net v0.55.0 // indirect golang.org/x/sys v0.45.0 // indirect + golang.org/x/text v0.37.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) + +replace github.com/Azure/agentbaker => ../ + +replace github.com/Azure/agentbaker/aks-live-patching => ../aks-live-patching + +replace github.com/coreos/ignition/v2 => github.com/flatcar/ignition/v2 v2.0.0-20250903113522-05b8a773288c diff --git a/aks-node-controller/go.sum b/aks-node-controller/go.sum index c58a6462885..4ab566d5771 100644 --- a/aks-node-controller/go.sum +++ b/aks-node-controller/go.sum @@ -2,23 +2,68 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1 h1:Wc1ml6QlJs2BHQ/9Bqu1jiyg github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM= github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE= github.com/Masterminds/semver/v3 v3.5.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/urfave/cli/v3 v3.8.0 h1:XqKPrm0q4P0q5JpoclYoCAv0/MIvH/jZ2umzuf8pNTI= github.com/urfave/cli/v3 v3.8.0/go.mod h1:ysVLtOEmg2tOy6PknnYVhDoouyC/6N42TMeoMzskhso= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A= -google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/aks-node-controller/localdnsconfig.go b/aks-node-controller/localdnsconfig.go new file mode 100644 index 00000000000..771d60b27c0 --- /dev/null +++ b/aks-node-controller/localdnsconfig.go @@ -0,0 +1,522 @@ +package main + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + akslivepatchingv1 "github.com/Azure/agentbaker/aks-live-patching/pkg/gen/akslivepatching/v1" + "github.com/Azure/agentbaker/aks-node-controller/helpers" + "github.com/Azure/agentbaker/aks-node-controller/parser" + aksnodeconfigv1 "github.com/Azure/agentbaker/aks-node-controller/pkg/gen/aksnodeconfig/v1" + "github.com/Azure/agentbaker/aks-node-controller/pkg/nodeconfigutils" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/encoding/protojson" +) + +const ( + localDNSLivePatchingComponentName = "localDNS" + defaultLocalDNSCorefilePath = "/opt/azure/containers/localdns/livepatched.localdns.corefile" + localDNSHostsFilePath = "/etc/localdns/hosts" + localDNSAgentPoolLabel = "kubernetes.azure.com/agentpool" + localDNSLPSALPNProto = "aks-live-patching" + localDNSALPNH2Proto = "h2" +) + +type localDNSConfigFetcher func(context.Context) (string, error) + +type localDNSConfigOutcome string + +const ( + outcomeLocalDNSConfigApplied localDNSConfigOutcome = "applied" + outcomeLocalDNSConfigAlreadyCurrent localDNSConfigOutcome = "alreadyCurrent" + outcomeLocalDNSConfigNotFound localDNSConfigOutcome = "notFound" + outcomeLocalDNSConfigNoCorefileData localDNSConfigOutcome = "noCorefileData" + outcomeLocalDNSConfigFailed localDNSConfigOutcome = "failed" +) + +type localDNSConfigPayload struct { + Corefile string `json:"corefile"` + CorefileBase64 string `json:"corefileBase64"` + CorefileBase64Alt string `json:"corefile_base64"` + CoreFile string `json:"coreFile"` + LocalDNSProfile json.RawMessage `json:"localDnsProfile"` + LocalDNSProfileAlt json.RawMessage `json:"local_dns_profile"` + AgentPools map[string]localDNSAgentPoolConfig `json:"agentPools"` + Profiles map[string]localDNSAgentPoolConfig `json:"profiles"` +} + +type localDNSAgentPoolConfig struct { + CorefileVersion string `json:"corefileVersion"` + ConfigChecksum string `json:"configChecksum"` + Corefile string `json:"corefile"` + CorefileBase64 string `json:"corefileBase64"` + CorefileBase64Alt string `json:"corefile_base64"` + CoreFile string `json:"coreFile"` + LocalDNSProfile json.RawMessage `json:"localDnsProfile"` + LocalDNSProfileAlt json.RawMessage `json:"local_dns_profile"` +} + +type localDNSCorefileUpdate struct { + corefile string + desiredVersion string + hasCorefile bool +} + +func (a *App) runApplyLocalDNSConfigCommand(ctx context.Context, configPath string, outputPath string, writer io.Writer) error { + config, err := readLocalDNSConfigInput(configPath) + if err != nil { + return err + } + outcome, err := a.applyLocalDNSConfig(ctx, config, outputPath) + if writer != nil { + _, _ = fmt.Fprintf(writer, "%s\n", outcome) + } + if err != nil { + return err + } + return nil +} + +func readLocalDNSConfigInput(configPath string) (string, error) { + if configPath == "" || configPath == "-" { + data, err := io.ReadAll(os.Stdin) + if err != nil { + return "", fmt.Errorf("reading localDNS config from stdin: %w", err) + } + return string(data), nil + } + data, err := os.ReadFile(configPath) + if err != nil { + return "", fmt.Errorf("reading localDNS config %s: %w", configPath, err) + } + return string(data), nil +} + +func (a *App) applyLocalDNSConfig(ctx context.Context, config string, outputPath string) (localDNSConfigOutcome, error) { + return a.fetchAndApplyLocalDNSConfigWithFetcher(ctx, outputPath, func(context.Context) (string, error) { + return config, nil + }) +} + +func (a *App) runFetchLocalDNSConfigCommand(ctx context.Context, outputPath string) (err error) { + slog.Info("aks-node-controller fetch-localdns-config started", "outputPath", outputPath) + startTime := time.Now() + defer func() { + if r := recover(); r != nil { + slog.Error("fetch-localdns-config panicked (fail-open)", "panic", r) + if a.eventLogger != nil { + a.eventLogger.LogEvent("FetchLocalDNSConfig", + fmt.Sprintf("fetch-localdns-config outcome=%s panic=%v", outcomeLocalDNSConfigFailed, r), + helpers.EventLevelError, startTime, time.Now()) + } + err = nil + } + }() + + outcome, err := a.fetchAndApplyLocalDNSConfig(ctx, outputPath) + level := helpers.EventLevelInformational + if outcome == outcomeLocalDNSConfigFailed { + level = helpers.EventLevelError + } + message := fmt.Sprintf("fetch-localdns-config outcome=%s", outcome) + if err != nil { + message = fmt.Sprintf("%s error=%s", message, err.Error()) + slog.Warn("fetch-localdns-config completed with error (fail-open)", "outcome", outcome, "error", err) + } else { + slog.Info("fetch-localdns-config completed", "outcome", outcome) + } + if a.eventLogger != nil { + a.eventLogger.LogEvent("FetchLocalDNSConfig", message, level, startTime, time.Now()) + } + return nil +} + +func (a *App) fetchAndApplyLocalDNSConfig(ctx context.Context, outputPath string) (localDNSConfigOutcome, error) { + return a.fetchAndApplyLocalDNSConfigWithFetcher(ctx, outputPath, a.fetchLocalDNSConfig) +} + +func (a *App) fetchAndApplyLocalDNSConfigWithFetcher(ctx context.Context, outputPath string, fetcher localDNSConfigFetcher) (localDNSConfigOutcome, error) { + if outputPath == "" { + outputPath = defaultLocalDNSCorefilePath + } + config, err := fetcher(ctx) + if err != nil { + if isLPSUnavailable(err) { + return outcomeLocalDNSConfigNotFound, nil + } + return outcomeLocalDNSConfigFailed, err + } + update, err := a.localDNSCorefileUpdateFromConfig(config) + if err != nil { + return outcomeLocalDNSConfigFailed, err + } + versionPath := localDNSCorefileVersionPath(outputPath) + if !update.hasCorefile { + return outcomeLocalDNSConfigNoCorefileData, nil + } + current, readErr := os.ReadFile(outputPath) + if readErr != nil && !os.IsNotExist(readErr) { + return outcomeLocalDNSConfigFailed, fmt.Errorf("reading localDNS corefile %s: %w", outputPath, readErr) + } + contentMatches := readErr == nil && bytes.Equal(current, []byte(update.corefile)) + if update.desiredVersion != "" { + currentVersion, err := readLocalDNSCorefileVersion(versionPath) + if err != nil { + return outcomeLocalDNSConfigFailed, err + } + if currentVersion == update.desiredVersion && contentMatches { + return outcomeLocalDNSConfigAlreadyCurrent, nil + } + } else if contentMatches { + return outcomeLocalDNSConfigAlreadyCurrent, nil + } + if err := writeLocalDNSCorefile(outputPath, update.corefile); err != nil { + return outcomeLocalDNSConfigFailed, err + } + if update.desiredVersion != "" { + if err := writeLocalDNSCorefileVersion(versionPath, update.desiredVersion); err != nil { + return outcomeLocalDNSConfigFailed, err + } + } + return outcomeLocalDNSConfigApplied, nil +} + +func (a *App) fetchLocalDNSConfig(ctx context.Context) (string, error) { + if a.fetchLocalDNSConfigFn != nil { + return a.fetchLocalDNSConfigFn(ctx) + } + return a.fetchLocalDNSConfigFromLPS(ctx) +} + +func (a *App) fetchLocalDNSConfigFromLPS(ctx context.Context) (string, error) { + fqdn, caPEM, err := a.lpsTargetFromNodeConfig() + if err != nil { + return "", fmt.Errorf("resolving LPS endpoint from node config: %w", err) + } + token, err := a.attestedToken(ctx) + if err != nil { + return "", fmt.Errorf("imds attested token: %w", err) + } + rootCAs, err := certPoolFromPEM(caPEM) + if err != nil { + return "", err + } + host := fqdn + if h, _, splitErr := net.SplitHostPort(fqdn); splitErr == nil { + host = h + } + target := net.JoinHostPort(host, lpsAPIServerPort) + tlsConfig := &tls.Config{ + MinVersion: tls.VersionTLS12, + RootCAs: rootCAs, + NextProtos: []string{localDNSLPSALPNProto, localDNSALPNH2Proto}, + InsecureSkipVerify: true, //nolint:gosec // SNI stays on the apiserver FQDN for ALPN routing; chain and hostname are verified below. + VerifyPeerCertificate: localDNSVerifyChainAgainstPool(rootCAs, lpsSNIHost), + } + ctx, cancel := context.WithTimeout(ctx, lpsFetchTimeout) + defer cancel() + + conn, err := grpc.NewClient( + target, + grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)), + ) + if err != nil { + return "", fmt.Errorf("creating LPS client: %w", err) + } + defer conn.Close() + + client := akslivepatchingv1.NewLivePatchingServiceClient(conn) + rpcCtx := metadata.AppendToOutgoingContext(ctx, "authorization", token) + resp, err := client.GetComponentConfig(rpcCtx, &akslivepatchingv1.GetComponentConfigRequest{ + ComponentName: localDNSLivePatchingComponentName, + }) + if err != nil { + if statusCode, ok := localDNSLPSUnavailableStatusCode(status.Code(err)); ok { + return "", &lpsUnavailableError{statusCode: statusCode} + } + return "", fmt.Errorf("get %s component config: %w", localDNSLivePatchingComponentName, err) + } + return resp.GetConfig(), nil +} + +func localDNSLPSUnavailableStatusCode(code codes.Code) (int, bool) { + switch code { + case codes.NotFound: + return http.StatusNotFound, true + case codes.PermissionDenied: + return http.StatusForbidden, true + case codes.Unauthenticated: + return http.StatusUnauthorized, true + default: + return 0, false + } +} + +func localDNSVerifyChainAgainstPool(pool *x509.CertPool, serverName string) func([][]byte, [][]*x509.Certificate) error { + return func(rawCerts [][]byte, _ [][]*x509.Certificate) error { + if len(rawCerts) == 0 { + return fmt.Errorf("server presented no certificates") + } + leaf, err := x509.ParseCertificate(rawCerts[0]) + if err != nil { + return fmt.Errorf("failed to parse server certificate: %w", err) + } + intermediates := x509.NewCertPool() + for _, raw := range rawCerts[1:] { + cert, err := x509.ParseCertificate(raw) + if err != nil { + return fmt.Errorf("failed to parse intermediate certificate: %w", err) + } + intermediates.AddCert(cert) + } + if _, err := leaf.Verify(x509.VerifyOptions{Roots: pool, Intermediates: intermediates, DNSName: serverName}); err != nil { + return fmt.Errorf("server certificate verification failed: %w", err) + } + return nil + } +} + +func certPoolFromPEM(caPEM []byte) (*x509.CertPool, error) { + if len(caPEM) == 0 { + return nil, fmt.Errorf("cluster CA unavailable from provision-config; refusing to fetch over unverified TLS") + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(caPEM) { + return nil, fmt.Errorf("failed to parse cluster CA PEM") + } + return pool, nil +} + +func (a *App) localDNSCorefileUpdateFromConfig(config string) (localDNSCorefileUpdate, error) { + config = strings.TrimSpace(config) + if config == "" { + return localDNSCorefileUpdate{}, nil + } + + var payload localDNSConfigPayload + if err := json.Unmarshal([]byte(config), &payload); err != nil { + return localDNSCorefileUpdate{}, fmt.Errorf("parsing localDNS LPS config: %w", err) + } + + selected, found, err := a.selectLocalDNSAgentPoolConfig(payload) + if err != nil || !found { + return localDNSCorefileUpdate{}, err + } + + update := localDNSCorefileUpdate{ + desiredVersion: firstNonEmpty(selected.CorefileVersion, selected.ConfigChecksum), + } + return a.localDNSCorefileUpdateFromAgentPoolConfig(selected, update) +} + +func (a *App) selectLocalDNSAgentPoolConfig(payload localDNSConfigPayload) (localDNSAgentPoolConfig, bool, error) { + selected := localDNSAgentPoolConfig{ + Corefile: payload.Corefile, + CorefileBase64: payload.CorefileBase64, + CorefileBase64Alt: payload.CorefileBase64Alt, + CoreFile: payload.CoreFile, + LocalDNSProfile: payload.LocalDNSProfile, + LocalDNSProfileAlt: payload.LocalDNSProfileAlt, + } + if len(payload.AgentPools) == 0 && len(payload.Profiles) == 0 { + return selected, true, nil + } + + agentPool, err := a.nodeAgentPoolName() + if err != nil { + return localDNSAgentPoolConfig{}, false, err + } + if selected, ok := payload.AgentPools[agentPool]; ok { + return selected, true, nil + } + if selected, ok := payload.Profiles[agentPool]; ok { + return selected, true, nil + } + return localDNSAgentPoolConfig{}, false, nil +} + +func (a *App) localDNSCorefileUpdateFromAgentPoolConfig(selected localDNSAgentPoolConfig, update localDNSCorefileUpdate) (localDNSCorefileUpdate, error) { + switch { + case strings.TrimSpace(selected.Corefile) != "": + update.corefile = selected.Corefile + update.hasCorefile = true + return update, nil + case strings.TrimSpace(selected.CoreFile) != "": + update.corefile = selected.CoreFile + update.hasCorefile = true + return update, nil + case strings.TrimSpace(selected.CorefileBase64) != "": + return update.withCorefileBase64(selected.CorefileBase64) + case strings.TrimSpace(selected.CorefileBase64Alt) != "": + return update.withCorefileBase64(selected.CorefileBase64Alt) + } + profileJSON := selected.LocalDNSProfile + if len(profileJSON) == 0 { + profileJSON = selected.LocalDNSProfileAlt + } + if len(profileJSON) == 0 { + if selected.CorefileVersion != "" || selected.ConfigChecksum != "" { + slog.Info("localDNS LPS config has only version/checksum; Corefile content is required for bootstrap mutation", + "corefileVersion", selected.CorefileVersion, "configChecksum", selected.ConfigChecksum) + } + return update, nil + } + profile := &aksnodeconfigv1.LocalDnsProfile{} + unmarshalOptions := protojson.UnmarshalOptions{DiscardUnknown: true} + if err := unmarshalOptions.Unmarshal(profileJSON, profile); err != nil { + return localDNSCorefileUpdate{}, fmt.Errorf("parsing localDNS profile: %w", err) + } + if !profile.GetEnableLocalDns() { + return update, nil + } + nodeConfig, err := a.nodeConfigWithLocalDNSProfile(profile) + if err != nil { + return localDNSCorefileUpdate{}, err + } + includeHostsPlugin := profile.GetEnableHostsPlugin() + if includeHostsPlugin { + if _, statErr := os.Stat(localDNSHostsFilePath); statErr != nil { + includeHostsPlugin = false + } + } + corefile, err := parser.GenerateLocalDNSCorefileFromAKSNodeConfig(nodeConfig, includeHostsPlugin) + if err != nil { + return localDNSCorefileUpdate{}, err + } + update.corefile = corefile + update.hasCorefile = true + return update, nil +} + +func (u localDNSCorefileUpdate) withCorefileBase64(v string) (localDNSCorefileUpdate, error) { + decoded, err := base64.StdEncoding.DecodeString(strings.TrimSpace(v)) + if err != nil { + return localDNSCorefileUpdate{}, fmt.Errorf("decoding localDNS corefileBase64: %w", err) + } + if len(strings.TrimSpace(string(decoded))) == 0 { + return u, nil + } + u.corefile = string(decoded) + u.hasCorefile = true + return u, nil +} + +func firstNonEmpty(values ...string) string { + for _, v := range values { + if strings.TrimSpace(v) != "" { + return strings.TrimSpace(v) + } + } + return "" +} + +func (a *App) nodeAgentPoolName() (string, error) { + path := a.getNodeConfigPath() + raw, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("reading node config %s: %w", path, err) + } + cfg, perr := nodeconfigutils.UnmarshalConfigurationV1(raw) + if perr != nil { + slog.Info("node config parsed with errors, continuing with partial config", "error", perr) + } + if cfg == nil { + return "", fmt.Errorf("node config %s could not be parsed", path) + } + agentPool := cfg.GetKubeletConfig().GetKubeletNodeLabels()[localDNSAgentPoolLabel] + if agentPool == "" { + return "", fmt.Errorf("node config has no %s kubelet node label", localDNSAgentPoolLabel) + } + return agentPool, nil +} + +func (a *App) nodeConfigWithLocalDNSProfile(profile *aksnodeconfigv1.LocalDnsProfile) (*aksnodeconfigv1.Configuration, error) { + path := a.getNodeConfigPath() + raw, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("reading node config %s: %w", path, err) + } + cfg, perr := nodeconfigutils.UnmarshalConfigurationV1(raw) + if perr != nil { + slog.Info("node config parsed with errors, continuing with partial config", "error", perr) + } + if cfg == nil { + return nil, fmt.Errorf("node config %s could not be parsed", path) + } + cfg.LocalDnsProfile = profile + return cfg, nil +} + +func writeLocalDNSCorefile(path string, corefile string) error { + if strings.TrimSpace(corefile) == "" { + return fmt.Errorf("localDNS corefile is empty") + } + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return fmt.Errorf("create parent directory: %w", err) + } + tmp, err := os.CreateTemp(filepath.Dir(path), ".localdns-corefile-*") + if err != nil { + return fmt.Errorf("creating temp file: %w", err) + } + tmpPath := tmp.Name() + if _, err := io.WriteString(tmp, corefile); err != nil { + tmp.Close() + os.Remove(tmpPath) + return fmt.Errorf("writing temp file: %w", err) + } + if err := tmp.Close(); err != nil { + os.Remove(tmpPath) + return fmt.Errorf("closing temp file: %w", err) + } + if err := os.Chmod(tmpPath, 0644); err != nil { + os.Remove(tmpPath) + return fmt.Errorf("chmod temp file: %w", err) + } + if err := os.Rename(tmpPath, path); err != nil { + os.Remove(tmpPath) + return fmt.Errorf("renaming temp file: %w", err) + } + return nil +} + +func localDNSCorefileVersionPath(corefilePath string) string { + return corefilePath + ".version" +} + +func readLocalDNSCorefileVersion(path string) (string, error) { + data, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return "", nil + } + return "", fmt.Errorf("reading localDNS corefile version %s: %w", path, err) + } + return strings.TrimSpace(string(data)), nil +} + +func writeLocalDNSCorefileVersion(path string, version string) error { + if strings.TrimSpace(version) == "" { + return nil + } + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return fmt.Errorf("create parent directory: %w", err) + } + return os.WriteFile(path, []byte(strings.TrimSpace(version)+"\n"), 0600) +} diff --git a/aks-node-controller/localdnsconfig_test.go b/aks-node-controller/localdnsconfig_test.go new file mode 100644 index 00000000000..ddfbf2beefe --- /dev/null +++ b/aks-node-controller/localdnsconfig_test.go @@ -0,0 +1,221 @@ +package main + +import ( + "context" + "encoding/base64" + "fmt" + "net/http" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" +) + +func writeLocalDNSTestNodeConfig(t *testing.T, app *App) { + t.Helper() + p := filepath.Join(t.TempDir(), "aks-node-controller-config.json") + require.NoError(t, os.WriteFile(p, []byte(fmt.Sprintf(`{ + "version": "v1", + "kubelet_config": { + "kubelet_node_labels": { + "kubernetes.azure.com/agentpool": %q + } + } +}`, "pool1")), 0o600)) + app.nodeConfigPath = p +} + +func TestFetchAndApplyLocalDNSConfig(t *testing.T) { + t.Run("corefileBase64 rewrites output", func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + want := ".:53 {\n forward . 168.63.129.16\n}\n" + tt.App.fetchLocalDNSConfigFn = func(context.Context) (string, error) { + return `{"corefileBase64":"` + base64.StdEncoding.EncodeToString([]byte(want)) + `"}`, nil + } + out := filepath.Join(t.TempDir(), "localdns.corefile") + + outcome, err := tt.App.fetchAndApplyLocalDNSConfig(context.Background(), out) + require.NoError(t, err) + assert.Equal(t, outcomeLocalDNSConfigApplied, outcome) + got, err := os.ReadFile(out) + require.NoError(t, err) + assert.Equal(t, want, string(got)) + }) + + t.Run("agent pool corefileBase64 rewrites output", func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + writeLocalDNSTestNodeConfig(t, tt.App) + want := ".:53 {\n forward . 168.63.129.16\n reload\n}\n" + tt.App.fetchLocalDNSConfigFn = func(context.Context) (string, error) { + return `{"agentPools":{"pool1":{"corefileVersion":"abc123","corefileBase64":"` + + base64.StdEncoding.EncodeToString([]byte(want)) + `"},"pool2":{"corefileBase64":"ignored"}}}`, nil + } + out := filepath.Join(t.TempDir(), "localdns.corefile") + + outcome, err := tt.App.fetchAndApplyLocalDNSConfig(context.Background(), out) + require.NoError(t, err) + assert.Equal(t, outcomeLocalDNSConfigApplied, outcome) + got, err := os.ReadFile(out) + require.NoError(t, err) + assert.Equal(t, want, string(got)) + version, err := os.ReadFile(localDNSCorefileVersionPath(out)) + require.NoError(t, err) + assert.Equal(t, "abc123\n", string(version)) + }) + + t.Run("already current version and content skips rewrite", func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + writeLocalDNSTestNodeConfig(t, tt.App) + out := filepath.Join(t.TempDir(), "localdns.corefile") + original := ".:53 {\n forward . 1.1.1.1\n}\n" + require.NoError(t, os.WriteFile(out, []byte(original), 0o644)) + require.NoError(t, writeLocalDNSCorefileVersion(localDNSCorefileVersionPath(out), "abc123")) + tt.App.fetchLocalDNSConfigFn = func(context.Context) (string, error) { + return `{"agentPools":{"pool1":{"corefileVersion":"abc123","corefileBase64":"` + + base64.StdEncoding.EncodeToString([]byte(original)) + `"}}}`, nil + } + + outcome, err := tt.App.fetchAndApplyLocalDNSConfig(context.Background(), out) + require.NoError(t, err) + assert.Equal(t, outcomeLocalDNSConfigAlreadyCurrent, outcome) + got, err := os.ReadFile(out) + require.NoError(t, err) + assert.Equal(t, original, string(got)) + }) + + t.Run("matching version with stale corefile rewrites output", func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + writeLocalDNSTestNodeConfig(t, tt.App) + out := filepath.Join(t.TempDir(), "localdns.corefile") + require.NoError(t, os.WriteFile(out, []byte("stale-corefile"), 0o644)) + require.NoError(t, writeLocalDNSCorefileVersion(localDNSCorefileVersionPath(out), "abc123")) + want := ".:53 {\n forward . 168.63.129.16\n}\n" + tt.App.fetchLocalDNSConfigFn = func(context.Context) (string, error) { + return `{"agentPools":{"pool1":{"corefileVersion":"abc123","corefileBase64":"` + + base64.StdEncoding.EncodeToString([]byte(want)) + `"}}}`, nil + } + + outcome, err := tt.App.fetchAndApplyLocalDNSConfig(context.Background(), out) + require.NoError(t, err) + assert.Equal(t, outcomeLocalDNSConfigApplied, outcome) + got, err := os.ReadFile(out) + require.NoError(t, err) + assert.Equal(t, want, string(got)) + }) + + t.Run("agent pool version only config is no-op", func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + writeLocalDNSTestNodeConfig(t, tt.App) + tt.App.fetchLocalDNSConfigFn = func(context.Context) (string, error) { + return `{"agentPools":{"pool1":{"corefileVersion":"abc123"}}}`, nil + } + out := filepath.Join(t.TempDir(), "localdns.corefile") + + outcome, err := tt.App.fetchAndApplyLocalDNSConfig(context.Background(), out) + require.NoError(t, err) + assert.Equal(t, outcomeLocalDNSConfigNoCorefileData, outcome) + _, statErr := os.Stat(out) + assert.True(t, os.IsNotExist(statErr)) + }) + + t.Run("agent pool localDnsProfile renders corefile", func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + writeLocalDNSTestNodeConfig(t, tt.App) + tt.App.fetchLocalDNSConfigFn = func(context.Context) (string, error) { + return `{ + "agentPools": { + "pool1": { + "corefileVersion": "profile-hash", + "localDnsProfile": { + "enableLocalDns": true, + "vnetDnsOverrides": { + ".": { + "queryLogging": "Error", + "protocol": "PreferUDP", + "forwardDestination": "VnetDNS", + "forwardPolicy": "Sequential", + "maxConcurrent": 1000, + "cacheDurationInSeconds": 3600, + "serveStaleDurationInSeconds": 3600, + "serveStale": "Immediate" + } + }, + "kubeDnsOverrides": { + "cluster.local": { + "queryLogging": "Error", + "protocol": "PreferUDP", + "forwardDestination": "ClusterCoreDNS", + "forwardPolicy": "Sequential", + "maxConcurrent": 1000, + "cacheDurationInSeconds": 3600, + "serveStaleDurationInSeconds": 3600, + "serveStale": "Immediate" + } + } + } + } + } +}`, nil + } + out := filepath.Join(t.TempDir(), "localdns.corefile") + + outcome, err := tt.App.fetchAndApplyLocalDNSConfig(context.Background(), out) + require.NoError(t, err) + assert.Equal(t, outcomeLocalDNSConfigApplied, outcome) + got, err := os.ReadFile(out) + require.NoError(t, err) + assert.Contains(t, string(got), "health-check.localdns.local:53") + assert.Contains(t, string(got), "cluster.local:53") + version, err := os.ReadFile(localDNSCorefileVersionPath(out)) + require.NoError(t, err) + assert.Equal(t, "profile-hash\n", string(version)) + }) + + t.Run("other agent pool config is no-op", func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + writeLocalDNSTestNodeConfig(t, tt.App) + tt.App.fetchLocalDNSConfigFn = func(context.Context) (string, error) { + return `{"agentPools":{"pool2":{"corefileBase64":"` + base64.StdEncoding.EncodeToString([]byte("ignored")) + `"}}}`, nil + } + out := filepath.Join(t.TempDir(), "localdns.corefile") + + outcome, err := tt.App.fetchAndApplyLocalDNSConfig(context.Background(), out) + require.NoError(t, err) + assert.Equal(t, outcomeLocalDNSConfigNoCorefileData, outcome) + _, statErr := os.Stat(out) + assert.True(t, os.IsNotExist(statErr)) + }) + + t.Run("cli action fails open", func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + tt.App.fetchLocalDNSConfigFn = func(context.Context) (string, error) { + return "", assert.AnError + } + exitCode := tt.App.Run(context.Background(), []string{"aks-node-controller", "fetch-localdns-config", "--output", filepath.Join(t.TempDir(), "localdns.corefile")}) + assert.Equal(t, 0, exitCode) + }) +} + +func TestLocalDNSLPSUnavailableStatusCode(t *testing.T) { + tests := []struct { + name string + code codes.Code + statusCode int + ok bool + }{ + {name: "not found", code: codes.NotFound, statusCode: http.StatusNotFound, ok: true}, + {name: "permission denied", code: codes.PermissionDenied, statusCode: http.StatusForbidden, ok: true}, + {name: "unauthenticated", code: codes.Unauthenticated, statusCode: http.StatusUnauthorized, ok: true}, + {name: "internal", code: codes.Internal, ok: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + statusCode, ok := localDNSLPSUnavailableStatusCode(tt.code) + assert.Equal(t, tt.ok, ok) + assert.Equal(t, tt.statusCode, statusCode) + }) + } +} diff --git a/aks-node-controller/parser/helper.go b/aks-node-controller/parser/helper.go index 9cc9e7eecca..0f348a40f42 100644 --- a/aks-node-controller/parser/helper.go +++ b/aks-node-controller/parser/helper.go @@ -894,7 +894,7 @@ type localDnsCorefileTemplateData struct { // Corefile is created using localdns.toml.gtpl template and aksnodeconfig values. // includeHostsPlugin controls whether the hosts plugin block is included in the generated Corefile. -func generateLocalDnsCorefileFromAKSNodeConfig(aksnodeconfig *aksnodeconfigv1.Configuration, includeHostsPlugin bool) (string, error) { +func GenerateLocalDNSCorefileFromAKSNodeConfig(aksnodeconfig *aksnodeconfigv1.Configuration, includeHostsPlugin bool) (string, error) { var corefileBuffer bytes.Buffer templateData := localDnsCorefileTemplateData{ Config: aksnodeconfig, @@ -906,6 +906,10 @@ func generateLocalDnsCorefileFromAKSNodeConfig(aksnodeconfig *aksnodeconfigv1.Co return corefileBuffer.String(), nil } +func generateLocalDnsCorefileFromAKSNodeConfig(aksnodeconfig *aksnodeconfigv1.Configuration, includeHostsPlugin bool) (string, error) { + return GenerateLocalDNSCorefileFromAKSNodeConfig(aksnodeconfig, includeHostsPlugin) +} + // getLocalDnsClusterListenerIp returns APIPA-IP address that will be used in localdns systemd unit. func getLocalDnsClusterListenerIp() string { return localDnsClusterListenerIp diff --git a/aks-node-controller/parser/helper_test.go b/aks-node-controller/parser/helper_test.go index 567431d225b..65382dbd732 100644 --- a/aks-node-controller/parser/helper_test.go +++ b/aks-node-controller/parser/helper_test.go @@ -1687,6 +1687,7 @@ health-check.localdns.local:53 { max_concurrent 1000 } ready 169.254.10.10:8181 + reload cache 3600 { success 9984 denial 9984 @@ -1714,6 +1715,7 @@ cluster.local:53 { max_concurrent 1000 } ready 169.254.10.10:8181 + reload cache 3600 { success 9984 denial 9984 @@ -1731,6 +1733,7 @@ testdomain456.com:53 { max_concurrent 1000 } ready 169.254.10.10:8181 + reload cache 3600 { success 9984 denial 9984 @@ -1756,6 +1759,7 @@ testdomain456.com:53 { max_concurrent 2000 } ready 169.254.10.11:8181 + reload cache 3600 { success 9984 denial 9984 diff --git a/aks-node-controller/parser/templates/containerd.toml.gtpl b/aks-node-controller/parser/templates/containerd.toml.gtpl index ecd492bee80..6b9d7442fdb 100644 --- a/aks-node-controller/parser/templates/containerd.toml.gtpl +++ b/aks-node-controller/parser/templates/containerd.toml.gtpl @@ -1,27 +1,12 @@ version = 2 oom_score = -999{{if getHasDataDir .KubeletConfig}} root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} -{{- if .GetIsKata }} -[plugins."io.containerd.snapshotter.v1.erofs"] - default_size = "10G" - enable_fsverity = false - ovl_mount_options = [] - max_unmerged_layers = 1 - -[plugins."io.containerd.service.v1.diff-service"] - default = ["erofs", "walking"] - -[plugins."io.containerd.differ.v1.erofs"] - mkfs_options = ["-T0", "--mkfs-time", "--sort=none"] - enable_tar_index = false -{{- end}} [plugins."io.containerd.grpc.v1.cri"] sandbox_image = "{{ .KubeBinaryConfig.GetPodInfraContainerImageUrl }}" enable_cdi = true [plugins."io.containerd.grpc.v1.cri".containerd] {{- if .GetIsKata }} disable_snapshot_annotations = false - snapshotter = "overlayfs" {{- end}} {{- if .GetEnableArtifactStreaming }} snapshotter = "overlaybd" @@ -77,7 +62,6 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} {{- if .GetIsKata }} [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata] runtime_type = "io.containerd.kata.v2" - snapshotter = "overlayfs" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.katacli] runtime_type = "io.containerd.runc.v1" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.katacli.options] @@ -90,12 +74,6 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} Root = "" CriuPath = "" SystemdCgroup = false -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview] - runtime_type = "io.containerd.kata.v2" - privileged_without_host_devices = true - snapshotter = "erofs" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview.options] - ConfigPath = "/usr/share/defaults/kata-containers/configuration-clh-templating.toml" [proxy_plugins] [proxy_plugins.tardev] type = "snapshot" diff --git a/aks-node-controller/parser/templates/containerd_no_GPU.toml.gtpl b/aks-node-controller/parser/templates/containerd_no_GPU.toml.gtpl index 90c52b01120..2eb27cab54c 100644 --- a/aks-node-controller/parser/templates/containerd_no_GPU.toml.gtpl +++ b/aks-node-controller/parser/templates/containerd_no_GPU.toml.gtpl @@ -1,26 +1,11 @@ version = 2 oom_score = -999{{if getHasDataDir .KubeletConfig}} root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} -{{- if .GetIsKata }} -[plugins."io.containerd.snapshotter.v1.erofs"] - default_size = "10G" - enable_fsverity = false - ovl_mount_options = [] - max_unmerged_layers = 1 - -[plugins."io.containerd.service.v1.diff-service"] - default = ["erofs", "walking"] - -[plugins."io.containerd.differ.v1.erofs"] - mkfs_options = ["-T0", "--mkfs-time", "--sort=none"] - enable_tar_index = false -{{- end}} [plugins."io.containerd.grpc.v1.cri"] sandbox_image = "{{ .KubeBinaryConfig.GetPodInfraContainerImageUrl }}" [plugins."io.containerd.grpc.v1.cri".containerd] {{- if .GetIsKata }} disable_snapshot_annotations = false - snapshotter = "overlayfs" {{- end}} {{- if .GetEnableArtifactStreaming }} snapshotter = "overlaybd" @@ -61,7 +46,6 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} {{- if .GetIsKata }} [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata] runtime_type = "io.containerd.kata.v2" - snapshotter = "overlayfs" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.katacli] runtime_type = "io.containerd.runc.v1" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.katacli.options] @@ -74,12 +58,6 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} Root = "" CriuPath = "" SystemdCgroup = false -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview] - runtime_type = "io.containerd.kata.v2" - privileged_without_host_devices = true - snapshotter = "erofs" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview.options] - ConfigPath = "/usr/share/defaults/kata-containers/configuration-clh-templating.toml" [proxy_plugins] [proxy_plugins.tardev] type = "snapshot" diff --git a/aks-node-controller/parser/templates/containerd_v2.toml.gtpl b/aks-node-controller/parser/templates/containerd_v2.toml.gtpl index 42442b8c57f..fd11c508145 100644 --- a/aks-node-controller/parser/templates/containerd_v2.toml.gtpl +++ b/aks-node-controller/parser/templates/containerd_v2.toml.gtpl @@ -1,20 +1,6 @@ version = 2 oom_score = -999{{if getHasDataDir .KubeletConfig}} root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} -{{- if .GetIsKata }} -[plugins."io.containerd.snapshotter.v1.erofs"] - default_size = "10G" - enable_fsverity = false - ovl_mount_options = [] - max_unmerged_layers = 1 - -[plugins."io.containerd.service.v1.diff-service"] - default = ["erofs", "walking"] - -[plugins."io.containerd.differ.v1.erofs"] - mkfs_options = ["-T0", "--mkfs-time", "--sort=none"] - enable_tar_index = false -{{- end}} [plugins."io.containerd.cri.v1.images"] {{- if .GetEnableArtifactStreaming }} snapshotter = "overlaybd" @@ -70,15 +56,8 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata] runtime_type = "io.containerd.kata.v2" privileged_without_host_devices = true - snapshotter = "overlayfs" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options] ConfigPath = "/usr/share/defaults/kata-containers/configuration.toml" -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview] - runtime_type = "io.containerd.kata.v2" - privileged_without_host_devices = true - snapshotter = "erofs" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview.options] - ConfigPath = "/usr/share/defaults/kata-containers/configuration-clh-templating.toml" [proxy_plugins] [proxy_plugins.tardev] type = "snapshot" diff --git a/aks-node-controller/parser/templates/containerd_v2_no_GPU.toml.gtpl b/aks-node-controller/parser/templates/containerd_v2_no_GPU.toml.gtpl index c28dd4f560c..45a2129f743 100644 --- a/aks-node-controller/parser/templates/containerd_v2_no_GPU.toml.gtpl +++ b/aks-node-controller/parser/templates/containerd_v2_no_GPU.toml.gtpl @@ -1,20 +1,6 @@ version = 2 oom_score = -999{{if getHasDataDir .KubeletConfig}} root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} -{{- if .GetIsKata }} -[plugins."io.containerd.snapshotter.v1.erofs"] - default_size = "10G" - enable_fsverity = false - ovl_mount_options = [] - max_unmerged_layers = 1 - -[plugins."io.containerd.service.v1.diff-service"] - default = ["erofs", "walking"] - -[plugins."io.containerd.differ.v1.erofs"] - mkfs_options = ["-T0", "--mkfs-time", "--sort=none"] - enable_tar_index = false -{{- end}} [plugins."io.containerd.cri.v1.images"] {{- if .GetEnableArtifactStreaming }} snapshotter = "overlaybd" @@ -57,15 +43,8 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata] runtime_type = "io.containerd.kata.v2" privileged_without_host_devices = true - snapshotter = "overlayfs" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options] ConfigPath = "/usr/share/defaults/kata-containers/configuration.toml" -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview] - runtime_type = "io.containerd.kata.v2" - privileged_without_host_devices = true - snapshotter = "erofs" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview.options] - ConfigPath = "/usr/share/defaults/kata-containers/configuration-clh-templating.toml" [proxy_plugins] [proxy_plugins.tardev] type = "snapshot" diff --git a/aks-node-controller/parser/templates/localdns.toml.gtpl b/aks-node-controller/parser/templates/localdns.toml.gtpl index 818b23aa421..88ee14118ca 100644 --- a/aks-node-controller/parser/templates/localdns.toml.gtpl +++ b/aks-node-controller/parser/templates/localdns.toml.gtpl @@ -45,8 +45,15 @@ health-check.localdns.local:53 { {{- end}} policy {{$forwardPolicy}} max_concurrent {{$override.MaxConcurrent}} + {{- if and $override.HealthCheck $override.HealthCheck.GetDuration}} + health_check {{$override.HealthCheck.GetDuration}}{{if $override.HealthCheck.GetNoRec}} no_rec{{end}}{{if $override.HealthCheck.GetDomain}} domain {{$override.HealthCheck.GetDomain}}{{end}} + {{- end}} + {{- if $override.GetFailfastAllUnhealthyUpstreams}} + failfast_all_unhealthy_upstreams + {{- end}} } ready {{getLocalDnsNodeListenerIp}}:8181 + reload cache {{$override.CacheDurationInSeconds}} { success 9984 denial 9984 @@ -78,7 +85,6 @@ health-check.localdns.local:53 { {{- range $domain, $override := $.Config.LocalDnsProfile.KubeDnsOverrides}} {{- $isRootDomain := eq $domain "." -}} {{- $fwdToClusterCoreDNS := or (hasSuffix $domain "cluster.local") (eq $override.ForwardDestination "ClusterCoreDNS")}} -{{- $forwardPolicy := "" }} {{- $forwardPolicy := "sequential" -}} {{- if eq $override.ForwardPolicy "RoundRobin" -}} {{- $forwardPolicy = "round_robin" -}} @@ -110,8 +116,15 @@ health-check.localdns.local:53 { {{- end}} policy {{$forwardPolicy}} max_concurrent {{$override.MaxConcurrent}} + {{- if and $override.HealthCheck $override.HealthCheck.GetDuration}} + health_check {{$override.HealthCheck.GetDuration}}{{if $override.HealthCheck.GetNoRec}} no_rec{{end}}{{if $override.HealthCheck.GetDomain}} domain {{$override.HealthCheck.GetDomain}}{{end}} + {{- end}} + {{- if $override.GetFailfastAllUnhealthyUpstreams}} + failfast_all_unhealthy_upstreams + {{- end}} } ready {{getLocalDnsClusterListenerIp}}:8181 + reload cache {{$override.CacheDurationInSeconds}} { success 9984 denial 9984 diff --git a/aks-node-controller/pkg/gen/aksnodeconfig/v1/localdns_config.pb.go b/aks-node-controller/pkg/gen/aksnodeconfig/v1/localdns_config.pb.go index d3981903347..95c2bb724d4 100644 --- a/aks-node-controller/pkg/gen/aksnodeconfig/v1/localdns_config.pb.go +++ b/aks-node-controller/pkg/gen/aksnodeconfig/v1/localdns_config.pb.go @@ -159,6 +159,10 @@ type LocalDnsOverrides struct { ServeStaleDurationInSeconds *int32 `protobuf:"varint,7,opt,name=serve_stale_duration_in_seconds,json=serveStaleDurationInSeconds,proto3,oneof" json:"serve_stale_duration_in_seconds,omitempty"` // Policy for serving stale data. ServeStale string `protobuf:"bytes,8,opt,name=serve_stale,json=serveStale,proto3" json:"serve_stale,omitempty"` + // Fail requests when all configured upstreams are unhealthy. + FailfastAllUnhealthyUpstreams *bool `protobuf:"varint,9,opt,name=failfast_all_unhealthy_upstreams,json=failfastAllUnhealthyUpstreams,proto3,oneof" json:"failfast_all_unhealthy_upstreams,omitempty"` + // Interval used by the forward plugin for upstream health checks, for example "500ms". + HealthCheck *LocalDnsHealthCheck `protobuf:"bytes,10,opt,name=health_check,json=healthCheck,proto3" json:"health_check,omitempty"` } func (x *LocalDnsOverrides) Reset() { @@ -247,6 +251,81 @@ func (x *LocalDnsOverrides) GetServeStale() string { return "" } +func (x *LocalDnsOverrides) GetFailfastAllUnhealthyUpstreams() bool { + if x != nil && x.FailfastAllUnhealthyUpstreams != nil { + return *x.FailfastAllUnhealthyUpstreams + } + return false +} + +func (x *LocalDnsOverrides) GetHealthCheck() *LocalDnsHealthCheck { + if x != nil { + return x.HealthCheck + } + return nil +} + +type LocalDnsHealthCheck struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Duration *string `protobuf:"bytes,1,opt,name=duration,proto3,oneof" json:"duration,omitempty"` + NoRec *bool `protobuf:"varint,2,opt,name=no_rec,json=noRec,proto3,oneof" json:"no_rec,omitempty"` + Domain *string `protobuf:"bytes,3,opt,name=domain,proto3,oneof" json:"domain,omitempty"` +} + +func (x *LocalDnsHealthCheck) Reset() { + *x = LocalDnsHealthCheck{} + mi := &file_aksnodeconfig_v1_localdns_config_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LocalDnsHealthCheck) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LocalDnsHealthCheck) ProtoMessage() {} + +func (x *LocalDnsHealthCheck) ProtoReflect() protoreflect.Message { + mi := &file_aksnodeconfig_v1_localdns_config_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LocalDnsHealthCheck.ProtoReflect.Descriptor instead. +func (*LocalDnsHealthCheck) Descriptor() ([]byte, []int) { + return file_aksnodeconfig_v1_localdns_config_proto_rawDescGZIP(), []int{2} +} + +func (x *LocalDnsHealthCheck) GetDuration() string { + if x != nil && x.Duration != nil { + return *x.Duration + } + return "" +} + +func (x *LocalDnsHealthCheck) GetNoRec() bool { + if x != nil && x.NoRec != nil { + return *x.NoRec + } + return false +} + +func (x *LocalDnsHealthCheck) GetDomain() string { + if x != nil && x.Domain != nil { + return *x.Domain + } + return "" +} + var File_aksnodeconfig_v1_localdns_config_proto protoreflect.FileDescriptor var file_aksnodeconfig_v1_localdns_config_proto_rawDesc = []byte{ @@ -307,7 +386,7 @@ var file_aksnodeconfig_v1_localdns_config_proto_rawDesc = []byte{ 0x6d, 0x6f, 0x72, 0x79, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x5f, 0x69, 0x6e, 0x5f, 0x6d, 0x62, 0x42, 0x2b, 0x0a, 0x29, 0x5f, 0x68, 0x6f, 0x73, 0x74, 0x73, 0x5f, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x5f, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, - 0x61, 0x6c, 0x5f, 0x69, 0x6e, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x22, 0xd9, 0x03, + 0x61, 0x6c, 0x5f, 0x69, 0x6e, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x22, 0x96, 0x05, 0x0a, 0x11, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x44, 0x6e, 0x73, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x71, 0x75, 0x65, 0x72, 0x79, 0x5f, 0x6c, 0x6f, 0x67, 0x67, 0x69, 0x6e, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x71, 0x75, 0x65, 0x72, @@ -332,18 +411,39 @@ var file_aksnodeconfig_v1_localdns_config_proto_rawDesc = []byte{ 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x88, 0x01, 0x01, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x65, 0x5f, 0x73, 0x74, 0x61, 0x6c, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x65, 0x72, 0x76, 0x65, 0x53, - 0x74, 0x61, 0x6c, 0x65, 0x42, 0x11, 0x0a, 0x0f, 0x5f, 0x6d, 0x61, 0x78, 0x5f, 0x63, 0x6f, 0x6e, - 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x42, 0x1c, 0x0a, 0x1a, 0x5f, 0x63, 0x61, 0x63, 0x68, - 0x65, 0x5f, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x6e, 0x5f, 0x73, 0x65, - 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x42, 0x22, 0x0a, 0x20, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x5f, - 0x73, 0x74, 0x61, 0x6c, 0x65, 0x5f, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, - 0x6e, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x42, 0x5a, 0x5a, 0x58, 0x67, 0x69, 0x74, - 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x2f, 0x61, 0x67, - 0x65, 0x6e, 0x74, 0x62, 0x61, 0x6b, 0x65, 0x72, 0x2f, 0x61, 0x6b, 0x73, 0x2d, 0x6e, 0x6f, 0x64, - 0x65, 0x2d, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2f, 0x70, 0x6b, 0x67, - 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x61, 0x6b, 0x73, 0x6e, 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x2f, 0x76, 0x31, 0x3b, 0x61, 0x6b, 0x73, 0x6e, 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x74, 0x61, 0x6c, 0x65, 0x12, 0x4c, 0x0a, 0x20, 0x66, 0x61, 0x69, 0x6c, 0x66, 0x61, 0x73, 0x74, + 0x5f, 0x61, 0x6c, 0x6c, 0x5f, 0x75, 0x6e, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x79, 0x5f, 0x75, + 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x48, 0x03, + 0x52, 0x1d, 0x66, 0x61, 0x69, 0x6c, 0x66, 0x61, 0x73, 0x74, 0x41, 0x6c, 0x6c, 0x55, 0x6e, 0x68, + 0x65, 0x61, 0x6c, 0x74, 0x68, 0x79, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x73, 0x88, + 0x01, 0x01, 0x12, 0x48, 0x0a, 0x0c, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x5f, 0x63, 0x68, 0x65, + 0x63, 0x6b, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x61, 0x6b, 0x73, 0x6e, 0x6f, + 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x6f, 0x63, 0x61, + 0x6c, 0x44, 0x6e, 0x73, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, + 0x0b, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x42, 0x11, 0x0a, 0x0f, + 0x5f, 0x6d, 0x61, 0x78, 0x5f, 0x63, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x42, + 0x1c, 0x0a, 0x1a, 0x5f, 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x5f, 0x69, 0x6e, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x42, 0x22, 0x0a, + 0x20, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x5f, 0x73, 0x74, 0x61, 0x6c, 0x65, 0x5f, 0x64, 0x75, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x6e, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, + 0x73, 0x42, 0x23, 0x0a, 0x21, 0x5f, 0x66, 0x61, 0x69, 0x6c, 0x66, 0x61, 0x73, 0x74, 0x5f, 0x61, + 0x6c, 0x6c, 0x5f, 0x75, 0x6e, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x79, 0x5f, 0x75, 0x70, 0x73, + 0x74, 0x72, 0x65, 0x61, 0x6d, 0x73, 0x22, 0x92, 0x01, 0x0a, 0x13, 0x4c, 0x6f, 0x63, 0x61, 0x6c, + 0x44, 0x6e, 0x73, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x1f, + 0x0a, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x48, 0x00, 0x52, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x88, 0x01, 0x01, 0x12, + 0x1a, 0x0a, 0x06, 0x6e, 0x6f, 0x5f, 0x72, 0x65, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x48, + 0x01, 0x52, 0x05, 0x6e, 0x6f, 0x52, 0x65, 0x63, 0x88, 0x01, 0x01, 0x12, 0x1b, 0x0a, 0x06, 0x64, + 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x02, 0x52, 0x06, 0x64, + 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x88, 0x01, 0x01, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x64, 0x75, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x6e, 0x6f, 0x5f, 0x72, 0x65, 0x63, + 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x42, 0x5a, 0x5a, 0x58, 0x67, + 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x2f, + 0x61, 0x67, 0x65, 0x6e, 0x74, 0x62, 0x61, 0x6b, 0x65, 0x72, 0x2f, 0x61, 0x6b, 0x73, 0x2d, 0x6e, + 0x6f, 0x64, 0x65, 0x2d, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2f, 0x70, + 0x6b, 0x67, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x61, 0x6b, 0x73, 0x6e, 0x6f, 0x64, 0x65, 0x63, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x2f, 0x76, 0x31, 0x3b, 0x61, 0x6b, 0x73, 0x6e, 0x6f, 0x64, 0x65, 0x63, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -358,23 +458,25 @@ func file_aksnodeconfig_v1_localdns_config_proto_rawDescGZIP() []byte { return file_aksnodeconfig_v1_localdns_config_proto_rawDescData } -var file_aksnodeconfig_v1_localdns_config_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_aksnodeconfig_v1_localdns_config_proto_msgTypes = make([]protoimpl.MessageInfo, 5) var file_aksnodeconfig_v1_localdns_config_proto_goTypes = []any{ - (*LocalDnsProfile)(nil), // 0: aksnodeconfig.v1.LocalDnsProfile - (*LocalDnsOverrides)(nil), // 1: aksnodeconfig.v1.LocalDnsOverrides - nil, // 2: aksnodeconfig.v1.LocalDnsProfile.VnetDnsOverridesEntry - nil, // 3: aksnodeconfig.v1.LocalDnsProfile.KubeDnsOverridesEntry + (*LocalDnsProfile)(nil), // 0: aksnodeconfig.v1.LocalDnsProfile + (*LocalDnsOverrides)(nil), // 1: aksnodeconfig.v1.LocalDnsOverrides + (*LocalDnsHealthCheck)(nil), // 2: aksnodeconfig.v1.LocalDnsHealthCheck + nil, // 3: aksnodeconfig.v1.LocalDnsProfile.VnetDnsOverridesEntry + nil, // 4: aksnodeconfig.v1.LocalDnsProfile.KubeDnsOverridesEntry } var file_aksnodeconfig_v1_localdns_config_proto_depIdxs = []int32{ - 2, // 0: aksnodeconfig.v1.LocalDnsProfile.vnet_dns_overrides:type_name -> aksnodeconfig.v1.LocalDnsProfile.VnetDnsOverridesEntry - 3, // 1: aksnodeconfig.v1.LocalDnsProfile.kube_dns_overrides:type_name -> aksnodeconfig.v1.LocalDnsProfile.KubeDnsOverridesEntry - 1, // 2: aksnodeconfig.v1.LocalDnsProfile.VnetDnsOverridesEntry.value:type_name -> aksnodeconfig.v1.LocalDnsOverrides - 1, // 3: aksnodeconfig.v1.LocalDnsProfile.KubeDnsOverridesEntry.value:type_name -> aksnodeconfig.v1.LocalDnsOverrides - 4, // [4:4] is the sub-list for method output_type - 4, // [4:4] is the sub-list for method input_type - 4, // [4:4] is the sub-list for extension type_name - 4, // [4:4] is the sub-list for extension extendee - 0, // [0:4] is the sub-list for field type_name + 3, // 0: aksnodeconfig.v1.LocalDnsProfile.vnet_dns_overrides:type_name -> aksnodeconfig.v1.LocalDnsProfile.VnetDnsOverridesEntry + 4, // 1: aksnodeconfig.v1.LocalDnsProfile.kube_dns_overrides:type_name -> aksnodeconfig.v1.LocalDnsProfile.KubeDnsOverridesEntry + 2, // 2: aksnodeconfig.v1.LocalDnsOverrides.health_check:type_name -> aksnodeconfig.v1.LocalDnsHealthCheck + 1, // 3: aksnodeconfig.v1.LocalDnsProfile.VnetDnsOverridesEntry.value:type_name -> aksnodeconfig.v1.LocalDnsOverrides + 1, // 4: aksnodeconfig.v1.LocalDnsProfile.KubeDnsOverridesEntry.value:type_name -> aksnodeconfig.v1.LocalDnsOverrides + 5, // [5:5] is the sub-list for method output_type + 5, // [5:5] is the sub-list for method input_type + 5, // [5:5] is the sub-list for extension type_name + 5, // [5:5] is the sub-list for extension extendee + 0, // [0:5] is the sub-list for field type_name } func init() { file_aksnodeconfig_v1_localdns_config_proto_init() } @@ -384,13 +486,14 @@ func file_aksnodeconfig_v1_localdns_config_proto_init() { } file_aksnodeconfig_v1_localdns_config_proto_msgTypes[0].OneofWrappers = []any{} file_aksnodeconfig_v1_localdns_config_proto_msgTypes[1].OneofWrappers = []any{} + file_aksnodeconfig_v1_localdns_config_proto_msgTypes[2].OneofWrappers = []any{} type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_aksnodeconfig_v1_localdns_config_proto_rawDesc, NumEnums: 0, - NumMessages: 4, + NumMessages: 5, NumExtensions: 0, NumServices: 0, }, diff --git a/aks-node-controller/proto/aksnodeconfig/v1/localdns_config.proto b/aks-node-controller/proto/aksnodeconfig/v1/localdns_config.proto index 8f11b898ee9..36e38e42785 100644 --- a/aks-node-controller/proto/aksnodeconfig/v1/localdns_config.proto +++ b/aks-node-controller/proto/aksnodeconfig/v1/localdns_config.proto @@ -62,4 +62,16 @@ message LocalDnsOverrides { // Policy for serving stale data. string serve_stale = 8; + + // Fail requests when all configured upstreams are unhealthy. + optional bool failfast_all_unhealthy_upstreams = 9; + + // Interval used by the forward plugin for upstream health checks, for example "500ms". + LocalDnsHealthCheck health_check = 10; +} + +message LocalDnsHealthCheck { + optional string duration = 1; + optional bool no_rec = 2; + optional string domain = 3; } diff --git a/e2e/config/vhd.go b/e2e/config/vhd.go index a79b7257e97..38a1f40df20 100644 --- a/e2e/config/vhd.go +++ b/e2e/config/vhd.go @@ -128,6 +128,18 @@ var ( Gallery: imageGalleryLinux, } + // VHDAzureLinuxV3Gen2Kata is the AzureLinux V3 Gen2 VHD built with FEATURE_FLAGS=kata. + // The image definition name mirrors the SIG_IMAGE_NAME produced by the VHD builder for + // OS_VERSION=V3kata + HYPERV_GENERATION=V2 (SKU_NAME=V3katagen2, prefixed with "AzureLinux"). + // See .pipelines/.vsts-vhd-builder-release.yaml (buildAzureLinuxV3gen2kata) and + // vhdbuilder/packer/produce-packer-settings-functions.sh (ensure_sig_image_name_linux). + VHDAzureLinuxV3Gen2Kata = &Image{ + Name: "AzureLinuxV3katagen2", + OS: OSAzureLinux, + Arch: "amd64", + Distro: datamodel.AKSAzureLinuxV3Gen2Kata, + Gallery: imageGalleryLinux, + } VHDAzureLinux3OSGuard = &Image{ Name: "AzureLinuxOSGuardOSGuardV3gen2fipsTL", OS: OSAzureLinux, diff --git a/e2e/exec.go b/e2e/exec.go index 260e3c8f627..7ca9c23b987 100644 --- a/e2e/exec.go +++ b/e2e/exec.go @@ -8,7 +8,6 @@ import ( "path/filepath" "strconv" "strings" - "sync" "time" scp "github.com/bramvdbogaerde/go-scp" @@ -19,12 +18,6 @@ import ( "k8s.io/client-go/tools/remotecommand" ) -var bufferPool = sync.Pool{ - New: func() any { - return new(bytes.Buffer) - }, -} - type podExecResult struct { exitCode string stderr, stdout string @@ -111,17 +104,26 @@ func runSSHCommandWithPrivateKeyFile( } defer session.Close() - stdout := bufferPool.Get().(*bytes.Buffer) - stderr := bufferPool.Get().(*bytes.Buffer) - stdout.Reset() - stderr.Reset() - - defer bufferPool.Put(stdout) - defer bufferPool.Put(stderr) - session.Stdout = stdout - session.Stderr = stderr - - err = session.Run(command) + var stdout bytes.Buffer + var stderr bytes.Buffer + session.Stdout = &stdout + session.Stderr = &stderr + + runErr := make(chan error, 1) + go func() { + runErr <- session.Run(command) + }() + select { + case err = <-runErr: + case <-ctx.Done(): + _ = session.Close() + _ = client.Close() + select { + case <-runErr: + case <-time.After(5 * time.Second): + } + return nil, fmt.Errorf("SSH command canceled or timed out: %w", ctx.Err()) + } exitCode := 0 if err != nil { diff --git a/e2e/kubelet/README.md b/e2e/kubelet/README.md index e7a48355ea3..01147aff586 100644 --- a/e2e/kubelet/README.md +++ b/e2e/kubelet/README.md @@ -131,7 +131,7 @@ map[string]string{ "--one-output": "\"false\"", "--oom-score-adj": "\"-999\"", "--pod-cidr": "\"\"", - "--pod-infra-container-image": "\"registry.k8s.io/pause:3.8\"", + "--pod-infra-container-image": "\"registry.k8s.io/pause:3.10.2\"", "--pod-manifest-path": "\"\"", "--pod-max-pids": "\"-1\"", "--pods-per-core": "\"0\"", diff --git a/e2e/node_config.go b/e2e/node_config.go index 6e161116b39..bc1aaa6b817 100644 --- a/e2e/node_config.go +++ b/e2e/node_config.go @@ -142,9 +142,11 @@ func getBaseNBC(ctx context.Context, t testing.TB, cluster *Cluster, vhd *config // is a temporary workaround // eventually we want to phase out usage of nbc -func nbcToAKSNodeConfigV1(nbc *datamodel.NodeBootstrappingConfiguration) *aksnodeconfigv1.Configuration { +func nbcToAKSNodeConfigV1(nbc *datamodel.NodeBootstrappingConfiguration) (*aksnodeconfigv1.Configuration, error) { cs := nbc.ContainerService - agent.ValidateAndSetLinuxNodeBootstrappingConfiguration(nbc) + if err := agent.ValidateAndSetLinuxNodeBootstrappingConfigurationWithError(nbc); err != nil { + return nil, err + } bootstrappingConfig := &aksnodeconfigv1.BootstrappingConfig{ TlsBootstrappingToken: nbc.KubeletClientTLSBootstrapToken, @@ -210,6 +212,12 @@ func nbcToAKSNodeConfigV1(nbc *datamodel.NodeBootstrappingConfiguration) *aksnod DisableCustomData: true, LinuxAdminUsername: "azureuser", VmSize: config.Config.DefaultVMSKU, + // The scriptless/aks-node-controller path gates its Kata containerd config blocks on + // this field alone, whereas the NBC/baker path derives Kata from the agent pool distro + // (see Distro.IsKataDistro and the IsKata template func in pkg/agent/baker.go). Without + // this mapping a scriptless scenario running on a Kata VHD would silently get a Kata + // image with a non-Kata containerd config. + IsKata: nbc.AgentPoolProfile.Distro.IsKataDistro(), ClusterConfig: &aksnodeconfigv1.ClusterConfig{ Location: nbc.ContainerService.Location, ResourceGroup: nbc.ResourceGroupName, @@ -377,7 +385,7 @@ func nbcToAKSNodeConfigV1(nbc *datamodel.NodeBootstrappingConfiguration) *aksnod cfg.KubeletConfig.KubeletFlags = kubeletFlags } - return cfg + return cfg, nil } // this is huge, but accurate, so leave it here. @@ -662,7 +670,7 @@ func baseTemplateLinux(t testing.TB, location string, k8sVersion string, arch st ContainerdDownloadURLBase: "https://storage.googleapis.com/cri-containerd-release/", CSIProxyDownloadURL: "https://packages.aks.azure.com/csi-proxy/v0.1.0/binaries/csi-proxy.tar.gz", WindowsProvisioningScriptsPackageURL: "https://packages.aks.azure.com/aks-engine/windows/provisioning/signedscripts-v0.2.2.zip", - WindowsPauseImageURL: "mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.1", + WindowsPauseImageURL: "mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.2", AlwaysPullWindowsPauseImage: false, CseScriptsPackageURL: "https://packages.aks.azure.com/aks/windows/cse/", CNIARM64PluginsDownloadURL: "https://packages.aks.azure.com/cni-plugins/v0.8.7/binaries/cni-plugins-linux-arm64-v0.8.7.tgz", @@ -674,7 +682,7 @@ func baseTemplateLinux(t testing.TB, location string, k8sVersion string, arch st OSImageConfig: map[datamodel.Distro]datamodel.AzureOSImageConfig(nil), }, K8sComponents: &datamodel.K8sComponents{ - PodInfraContainerImageURL: "mcr.microsoft.com/oss/v2/kubernetes/pause:3.6", + PodInfraContainerImageURL: "mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.2", HyperkubeImageURL: "mcr.microsoft.com/oss/kubernetes/", WindowsPackageURL: "windowspackage", LinuxCredentialProviderURL: "", @@ -936,7 +944,7 @@ func baseTemplateWindows(t testing.TB, location string) *datamodel.NodeBootstrap WindowsDockerVersion: "", WindowsImageSourceURL: "", WindowsOffer: "aks-windows", - WindowsPauseImageURL: "mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.1", + WindowsPauseImageURL: "mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.2", WindowsPublisher: "microsoft-aks", WindowsSku: "", }, @@ -979,7 +987,7 @@ DXRqvV7TWO2hndliQq3BW385ZkiephlrmpUVM= r2k1@arturs-mbp.lan`, // VnetCNIARM64LinuxPluginsDownloadURL: "https://packages.aks.azure.com/azure-cni/v1.4.13/binaries/azure-vnet-cni-linux-arm64-v1.4.14.tgz", // VnetCNILinuxPluginsDownloadURL: "https://packages.aks.azure.com/azure-cni/v1.1.3/binaries/azure-vnet-cni-linux-amd64-v1.1.3.tgz", VnetCNIWindowsPluginsDownloadURL: "https://packages.aks.azure.com/azure-cni/v1.6.21/binaries/azure-vnet-cni-windows-amd64-v1.6.21.zip", - WindowsPauseImageURL: "mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.1", + WindowsPauseImageURL: "mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.2", WindowsProvisioningScriptsPackageURL: "", WindowsTelemetryGUID: "fb801154-36b9-41bc-89c2-f4d4f05472b0", }, diff --git a/e2e/scenario_localdns_hosts_test.go b/e2e/scenario_localdns_hosts_test.go index c83eba792a9..8db8dab1bf7 100644 --- a/e2e/scenario_localdns_hosts_test.go +++ b/e2e/scenario_localdns_hosts_test.go @@ -1,12 +1,31 @@ package e2e import ( + "bytes" + "compress/gzip" + "context" + "encoding/base64" + "os" + "strings" "testing" + "time" aksnodeconfigv1 "github.com/Azure/agentbaker/aks-node-controller/pkg/gen/aksnodeconfig/v1" "github.com/Azure/agentbaker/e2e/config" "github.com/Azure/agentbaker/pkg/agent/datamodel" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v7" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" +) + +const ( + desiredLocalDNSVersion = "e2e-localdns-corefile-version" + localDNSPayloadPath = "/opt/azure/containers/localdns/e2e-localdns-lps-payload.json" + localDNSFetcherStamp = "/opt/azure/containers/localdns/e2e-localdns-lps-fetcher-called" + localDNSBranchScriptArchivePath = "/opt/azure/containers/localdns/e2e-localdns.sh.gz.b64" + localDNSFetcherPath = "/opt/azure/containers/localdns/e2e-fetch-localdns-config" + localDNSUnavailableFetcherStamp = "/opt/azure/containers/localdns/e2e-localdns-lps-unavailable-called" ) // Test_LocalDNSHostsPlugin tests the localdns hosts plugin across all supported distros @@ -56,3 +75,283 @@ func Test_LocalDNSHostsPlugin(t *testing.T) { }) } } + +// Test_LocalDNSLPSBootstrapPatch validates the node-side LocalDNS live-patching +// bootstrap path. It simulates LPS by temporarily wrapping aks-node-controller's +// fetch-localdns-config command so it returns a LocalDNS nodeConfig payload, then +// delegates to the real apply-localdns-config implementation. The test verifies that +// localdns.sh: +// 1. invokes the fetcher before CoreDNS starts, +// 2. renders the supplied LocalDNS profile payload into updated.localdns.corefile, +// 3. persists the paired corefileVersion, and +// 4. stamps components.localDNS.current after kubeconfig/node registration. +func Test_LocalDNSLPSBootstrapPatch(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests LocalDNS LPS bootstrap patching applies Corefile and reports corefileVersion", + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2404Gen2Containerd, + // Force compiling the local aks-node-controller so the provision-config parser matches + // the baker-generated nbc-cmd for Corefile env vars (compareEnvs parity). + ForceScriptlessCompilation: true, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.AgentPoolProfile.LocalDNSProfile.EnableLocalDNS = true + }, + CustomDataWriteFiles: []CustomDataWriteFile{ + { + Path: localDNSBranchScriptArchivePath, + Permissions: "0644", + Owner: "root", + Content: mustReadCompressedLocalDNSArtifact(t), + }, + { + Path: "/etc/systemd/system/localdns.service.d/00-e2e-branch-localdns.conf", + Permissions: "0644", + Owner: "root", + Content: localDNSBranchScriptDropIn(), + }, + { + Path: localDNSPayloadPath, + Permissions: "0644", + Owner: "root", + Content: localDNSLPSPayload(desiredLocalDNSVersion), + }, + { + Path: localDNSFetcherPath, + Permissions: "0755", + Owner: "root", + Content: localDNSLPSFetcherWrapper(), + }, + }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) { + config.LocalDnsProfile.EnableLocalDns = true + }, + Validator: validateLocalDNSLPSBootstrapPatch, + }, + }) +} + +// Test_LocalDNSLPSUnavailableFallback validates the unhappy bootstrap path: when LPS +// has no LocalDNS config published for the node (fetch-localdns-config fails open with +// no livepatched Corefile written), localdns.sh must fall back to the baked/CSE-generated +// localdns.corefile and CoreDNS must still come up and serve DNS. This is the failure-mode +// counterpart to Test_LocalDNSLPSBootstrapPatch, exercised end-to-end on a live node. +// +// The test verifies that: +// 1. the fetcher was invoked (LPS was consulted), +// 2. no livepatched Corefile or version file is written, +// 3. updated.localdns.corefile is still produced (from the baked source), +// 4. localdns.service is enabled and resolves DNS, and +// 5. the node does NOT report a LocalDNS live-patching current version. +func Test_LocalDNSLPSUnavailableFallback(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests LocalDNS bootstrap falls back to the baked Corefile when LPS has no config", + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2404Gen2Containerd, + // Force compiling the local aks-node-controller so localdns.sh under test matches the + // branch's Corefile generation (compareEnvs parity), same as the happy-path scenario. + ForceScriptlessCompilation: true, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.AgentPoolProfile.LocalDNSProfile.EnableLocalDNS = true + }, + CustomDataWriteFiles: []CustomDataWriteFile{ + { + Path: localDNSBranchScriptArchivePath, + Permissions: "0644", + Owner: "root", + Content: mustReadCompressedLocalDNSArtifact(t), + }, + { + Path: "/etc/systemd/system/localdns.service.d/00-e2e-branch-localdns.conf", + Permissions: "0644", + Owner: "root", + Content: localDNSBranchScriptDropIn(), + }, + { + Path: localDNSFetcherPath, + Permissions: "0755", + Owner: "root", + Content: localDNSLPSUnavailableFetcherWrapper(), + }, + }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) { + config.LocalDnsProfile.EnableLocalDns = true + }, + Validator: validateLocalDNSLPSUnavailableFallback, + }, + }) +} + +func mustReadCompressedLocalDNSArtifact(t *testing.T) string { + t.Helper() + data, err := os.ReadFile("../parts/linux/cloud-init/artifacts/localdns.sh") + require.NoError(t, err) + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + _, err = zw.Write(data) + require.NoError(t, err) + require.NoError(t, zw.Close()) + content := strings.ReplaceAll(string(data), `AKS_NODE_CONTROLLER_BINARY="/opt/azure/containers/aks-node-controller"`, `AKS_NODE_CONTROLLER_BINARY="`+localDNSFetcherPath+`"`) + buf.Reset() + zw = gzip.NewWriter(&buf) + _, err = zw.Write([]byte(content)) + require.NoError(t, err) + require.NoError(t, zw.Close()) + return base64.StdEncoding.EncodeToString(buf.Bytes()) +} + +func localDNSBranchScriptDropIn() string { + // LOCALDNS_ENABLE_LEGACY_LIVEPATCH_STATUS makes localdns.sh write the + // live-patching-status node annotation itself. In production the knead + // live-patching loop owns this annotation, but knead does not drive the + // localDNS component in this E2E, so opt into the bootstrap writer here. + return `[Service] +Environment="LOCALDNS_ENABLE_LEGACY_LIVEPATCH_STATUS=true" +ExecStartPre=/bin/bash -c 'base64 -d ` + localDNSBranchScriptArchivePath + ` | gzip -d > /opt/azure/containers/localdns/localdns.sh && chmod 0544 /opt/azure/containers/localdns/localdns.sh' +` +} + +func localDNSLPSPayload(version string) string { + return `{ + "agentPools": { + "nodepool2": { + "corefileVersion": "` + version + `", + "localDnsProfile": { + "enableLocalDns": true, + "vnetDnsOverrides": { + ".": { + "queryLogging": "Error", + "protocol": "PreferUDP", + "forwardDestination": "VnetDNS", + "forwardPolicy": "Sequential", + "maxConcurrent": 1000, + "cacheDurationInSeconds": 3600, + "serveStaleDurationInSeconds": 3600, + "serveStale": "Immediate" + } + }, + "kubeDnsOverrides": { + "cluster.local": { + "queryLogging": "Error", + "protocol": "PreferUDP", + "forwardDestination": "ClusterCoreDNS", + "forwardPolicy": "Sequential", + "maxConcurrent": 1000, + "cacheDurationInSeconds": 3600, + "serveStaleDurationInSeconds": 3600, + "serveStale": "Immediate" + } + } + } + } + } +}` +} + +func localDNSLPSFetcherWrapper() string { + return `#!/bin/bash +set -euo pipefail +# Prefer the locally-compiled aks-node-controller (delivered via the hotfix path by +# ForceScriptlessCompilation) so apply-localdns-config matches the branch under test; +# fall back to the VHD baked-in binary otherwise. +ANC_BIN=/opt/azure/containers/aks-node-controller +if [ -x /opt/azure/containers/aks-node-controller-hotfix ]; then + ANC_BIN=/opt/azure/containers/aks-node-controller-hotfix +fi +if [ "${1:-}" != "fetch-localdns-config" ]; then + exec "$ANC_BIN" "$@" +fi +output="" +while [ "$#" -gt 0 ]; do + case "$1" in + --output) + output="$2" + shift 2 + ;; + *) + shift + ;; + esac +done +if [ -z "$output" ]; then + echo "missing --output" >&2 + exit 1 +fi +touch ` + localDNSFetcherStamp + ` +exec "$ANC_BIN" apply-localdns-config --config-file ` + localDNSPayloadPath + ` --output "$output" +` +} + +// localDNSLPSUnavailableFetcherWrapper simulates LPS having no LocalDNS config for the node. +// It records that fetch-localdns-config was invoked, then exits 0 without writing the output +// Corefile -- the same fail-open behavior aks-node-controller exhibits when LPS returns a +// benign "unavailable" status (NotFound/PermissionDenied/Unauthenticated). Non-fetch +// subcommands are delegated to the real binary so the rest of provisioning is unaffected. +func localDNSLPSUnavailableFetcherWrapper() string { + return `#!/bin/bash +set -euo pipefail +ANC_BIN=/opt/azure/containers/aks-node-controller +if [ -x /opt/azure/containers/aks-node-controller-hotfix ]; then + ANC_BIN=/opt/azure/containers/aks-node-controller-hotfix +fi +if [ "${1:-}" != "fetch-localdns-config" ]; then + exec "$ANC_BIN" "$@" +fi +# LPS has nothing published for this node: record the call and fail open without writing a +# livepatched Corefile, so localdns.sh falls back to the baked localdns.corefile. +touch ` + localDNSUnavailableFetcherStamp + ` +exit 0 +` +} + +func validateLocalDNSLPSBootstrapPatch(ctx context.Context, s *Scenario) { + const ( + updatedCorefile = "/opt/azure/containers/localdns/updated.localdns.corefile" + livepatchedCorefile = "/opt/azure/containers/localdns/livepatched.localdns.corefile" + ) + + ValidateFileExists(ctx, s, localDNSFetcherStamp) + ValidateFileHasContent(ctx, s, updatedCorefile, "health-check.localdns.local:53") + ValidateFileHasContent(ctx, s, updatedCorefile, "cluster.local:53") + ValidateFileHasContent(ctx, s, livepatchedCorefile+".version", desiredLocalDNSVersion) + ValidateLocalDNSService(ctx, s, "enabled") + ValidateLocalDNSResolution(ctx, s, "169.254.10.10") + + err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 3*time.Minute, true, func(ctx context.Context) (bool, error) { + node, err := s.Runtime.Kube.Typed.CoreV1().Nodes().Get(ctx, s.Runtime.VM.KubeName, metav1.GetOptions{}) + if err != nil { + return false, nil + } + status := node.Annotations["kubernetes.azure.com/live-patching-status"] + return strings.Contains(status, `"localDNS":{"current":"`+desiredLocalDNSVersion+`"}`), nil + }) + require.NoError(s.T, err, "node did not report LocalDNS live-patching current version %q", desiredLocalDNSVersion) +} + +func validateLocalDNSLPSUnavailableFallback(ctx context.Context, s *Scenario) { + const ( + bakedCorefile = "/opt/azure/containers/localdns/localdns.corefile" + updatedCorefile = "/opt/azure/containers/localdns/updated.localdns.corefile" + livepatchedCorefile = "/opt/azure/containers/localdns/livepatched.localdns.corefile" + ) + + // The fetcher ran (LPS was consulted) but wrote nothing, so no livepatched Corefile + // or version file should exist and localdns.sh must fall back to the baked Corefile. + ValidateFileExists(ctx, s, localDNSUnavailableFetcherStamp) + ValidateFileDoesNotExist(ctx, s, livepatchedCorefile) + ValidateFileDoesNotExist(ctx, s, livepatchedCorefile+".version") + + // CoreDNS still comes up from the baked source and serves DNS. + ValidateFileExists(ctx, s, bakedCorefile) + ValidateFileHasContent(ctx, s, updatedCorefile, "health-check.localdns.local:53") + ValidateLocalDNSService(ctx, s, "enabled") + ValidateLocalDNSResolution(ctx, s, "169.254.10.10") + + // No LocalDNS live-patching version should be reported when LPS had nothing to apply. + node, err := s.Runtime.Kube.Typed.CoreV1().Nodes().Get(ctx, s.Runtime.VM.KubeName, metav1.GetOptions{}) + require.NoError(s.T, err, "failed to get node %s", s.Runtime.VM.KubeName) + status := node.Annotations["kubernetes.azure.com/live-patching-status"] + require.NotContains(s.T, status, `"localDNS"`, + "node unexpectedly reported a LocalDNS live-patching status when LPS had no config: %q", status) +} diff --git a/e2e/scenario_test.go b/e2e/scenario_test.go index dc6fc03302f..aac3b224835 100644 --- a/e2e/scenario_test.go +++ b/e2e/scenario_test.go @@ -411,6 +411,45 @@ func Test_AzureLinuxV3(t *testing.T) { }) } +// Test_AzureLinuxV3Gen2Kata verifies that AgentBaker correctly bootstraps a Kata-enabled node. +// +// Kata Containers is a runtime, so the thing that can silently break is the containerd +// configuration: pkg/agent/baker.go only emits the `kata` runtime handler blocks +// when the agent pool's Distro satisfies Distro.IsKataDistro(). Selecting a Kata VHD here flows +// through e2e/node_config.go -> AgentPoolProfile.Distro -> the IsKata template func, so this +// scenario exercises that whole path against a real node. +// +// The scenario asserts three increasingly strong properties: +// 1. the rendered /etc/containerd/config.toml contains the Kata runtime handlers, +// 2. containerd actually parsed and loaded them (no warnings, handlers in `config dump`), +// 3. for every handler in kataRuntimeHandlers, a pod scheduled via a Kata RuntimeClass runs +// and is genuinely VM-isolated. +func Test_AzureLinuxV3Gen2Kata(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that an AzureLinuxV3 Gen2 Kata node is bootstrapped with working kata containerd runtime handlers, and can run VM-isolated pods via Kata RuntimeClasses", + Tags: Tags{ + Kata: true, + }, + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDAzureLinuxV3Gen2Kata, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + // Leave unattended upgrades on so that CSE's kata-specific opt-out branch is + // actually exercised, which ValidateKataHostReadiness asserts. + nbc.DisableUnattendedUpgrades = false + }, + Validator: func(ctx context.Context, s *Scenario) { + ValidateKataContainerdConfig(ctx, s) + ValidateKataContainerdConfigDump(ctx, s) + ValidateKataHostReadiness(ctx, s) + for _, handler := range kataRuntimeHandlers { + ValidateKataPodIsIsolated(ctx, s, handler) + } + }, + }, + }) +} + func Test_AzureLinuxV3_CustomCA(t *testing.T) { RunScenario(t, &Scenario{ Description: "Tests that an AzureLinuxV3 node can be properly bootstrapped with custom ca", @@ -1631,6 +1670,76 @@ func Test_AzureLinuxV3_CustomSysctls(t *testing.T) { }) } +func Test_AzureLinuxV3_CustomLinuxOSConfigPersistsAfterReboot(t *testing.T) { + customSysctls := map[string]string{ + "net.ipv4.ip_local_port_range": "32768 62535", + "net.netfilter.nf_conntrack_max": "2097152", + "net.netfilter.nf_conntrack_buckets": "524288", + "net.ipv4.tcp_keepalive_intvl": "90", + } + customContainerdUlimits := map[string]string{ + "LimitMEMLOCK": "75000", + "LimitNOFILE": "1048", + } + const ( + swapFileSizeMB int32 = 64 + thpEnabled = "never" + thpDefrag = "never" + ) + + RunScenario(t, &Scenario{ + Description: "tests that AzureLinuxV3 custom Linux OS config persists after a node reboot", + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDAzureLinuxV3Gen2, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.AgentPoolProfile.CustomLinuxOSConfig = &datamodel.CustomLinuxOSConfig{ + Sysctls: &datamodel.SysctlConfig{ + NetNetfilterNfConntrackMax: to.Ptr(toolkit.StrToInt32(customSysctls["net.netfilter.nf_conntrack_max"])), + NetNetfilterNfConntrackBuckets: to.Ptr(toolkit.StrToInt32(customSysctls["net.netfilter.nf_conntrack_buckets"])), + NetIpv4IpLocalPortRange: customSysctls["net.ipv4.ip_local_port_range"], + NetIpv4TcpkeepaliveIntvl: to.Ptr(toolkit.StrToInt32(customSysctls["net.ipv4.tcp_keepalive_intvl"])), + }, + UlimitConfig: &datamodel.UlimitConfig{ + MaxLockedMemory: customContainerdUlimits["LimitMEMLOCK"], + NoFile: customContainerdUlimits["LimitNOFILE"], + }, + SwapFileSizeMB: to.Ptr(swapFileSizeMB), + TransparentHugePageEnabled: thpEnabled, + TransparentHugePageDefrag: thpDefrag, + } + nbc.AgentPoolProfile.CustomKubeletConfig = &datamodel.CustomKubeletConfig{ + FailSwapOn: to.Ptr(false), + } + }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) { + config.CustomLinuxOsConfig = &aksnodeconfigv1.CustomLinuxOsConfig{ + SysctlConfig: &aksnodeconfigv1.SysctlConfig{ + NetNetfilterNfConntrackMax: to.Ptr(toolkit.StrToInt32(customSysctls["net.netfilter.nf_conntrack_max"])), + NetNetfilterNfConntrackBuckets: to.Ptr(toolkit.StrToInt32(customSysctls["net.netfilter.nf_conntrack_buckets"])), + NetIpv4IpLocalPortRange: to.Ptr(customSysctls["net.ipv4.ip_local_port_range"]), + NetIpv4TcpkeepaliveIntvl: to.Ptr(toolkit.StrToInt32(customSysctls["net.ipv4.tcp_keepalive_intvl"])), + }, + UlimitConfig: &aksnodeconfigv1.UlimitConfig{ + MaxLockedMemory: to.Ptr(customContainerdUlimits["LimitMEMLOCK"]), + NoFile: to.Ptr(customContainerdUlimits["LimitNOFILE"]), + }, + EnableSwapConfig: true, + SwapFileSize: swapFileSizeMB, + TransparentHugepageSupport: thpEnabled, + TransparentDefrag: thpDefrag, + } + config.KubeletConfig.EnableKubeletConfigFile = true + config.KubeletConfig.KubeletConfigFileConfig.FailSwapOn = to.Ptr(false) + }, + WaitForSSHAfterReboot: 10 * time.Minute, + Validator: func(ctx context.Context, s *Scenario) { + ValidateCustomLinuxOSConfigPersistsAfterReboot(ctx, s, customSysctls, customContainerdUlimits, swapFileSizeMB, thpEnabled, thpDefrag) + }, + }, + }) +} + func Test_Ubuntu2204_KubeletCustomConfig(t *testing.T) { RunScenario(t, &Scenario{ Tags: Tags{ diff --git a/e2e/scenario_win_test.go b/e2e/scenario_win_test.go index 4eb18adb7f8..fe7064e2c3a 100644 --- a/e2e/scenario_win_test.go +++ b/e2e/scenario_win_test.go @@ -66,6 +66,7 @@ func Test_Windows2022_AzureNetwork(t *testing.T) { ValidateWindowsDisplayVersion(ctx, s, "21H2") ValidateFileHasContent(ctx, s, "/k/kubeletstart.ps1", "--container-runtime=remote") ValidateKubeletArgs(ctx, s) + ValidateContainerdWindowsPriorityClass(ctx, s) ValidateCiliumIsNotRunningWindows(ctx, s) ValidateDotnetNotInstalledWindows(ctx, s) ValidateWindowsSystemServicesRestartConfiguration(ctx, s) @@ -91,6 +92,7 @@ func Test_Windows2022AzureOverlayNetworkDualStack(t *testing.T) { ValidateWindowsDisplayVersion(ctx, s, "21H2") ValidateFileHasContent(ctx, s, "/k/kubeletstart.ps1", "--container-runtime=remote") ValidateKubeletArgs(ctx, s) + ValidateContainerdWindowsPriorityClass(ctx, s) ValidateCiliumIsNotRunningWindows(ctx, s) ValidateWindowsSystemServicesRestartConfiguration(ctx, s) ValidateCollectWindowsLogsScript(ctx, s) @@ -113,6 +115,7 @@ func Test_Windows2022Gen2AzureNetwork(t *testing.T) { ValidateWindowsDisplayVersion(ctx, s, "21H2") ValidateFileHasContent(ctx, s, "/k/kubeletstart.ps1", "--container-runtime=remote") ValidateKubeletArgs(ctx, s) + ValidateContainerdWindowsPriorityClass(ctx, s) ValidateCiliumIsNotRunningWindows(ctx, s) ValidateDotnetNotInstalledWindows(ctx, s) ValidateFileHasContent(ctx, s, "/AzureData/CustomDataSetupScript.log", "CSEScriptsPackageUrl used for provision is https://packages.aks.azure.com/aks/windows/cse/aks-windows-cse-scripts-current.zip") @@ -139,6 +142,7 @@ func Test_Windows2022Gen2AzureOverlayNetworkDualStack(t *testing.T) { ValidateWindowsDisplayVersion(ctx, s, "21H2") ValidateFileHasContent(ctx, s, "/k/kubeletstart.ps1", "--container-runtime=remote") ValidateKubeletArgs(ctx, s) + ValidateContainerdWindowsPriorityClass(ctx, s) ValidateCiliumIsNotRunningWindows(ctx, s) ValidateFileHasContent(ctx, s, "/AzureData/CustomDataSetupScript.log", "CSEScriptsPackageUrl used for provision is https://packages.aks.azure.com/aks/windows/cse/aks-windows-cse-scripts-current.zip") ValidateWindowsSystemServicesRestartConfiguration(ctx, s) @@ -183,6 +187,7 @@ func Test_Windows2025(t *testing.T) { ValidateWindowsDisplayVersion(ctx, s, "24H2") ValidateFileHasContent(ctx, s, "/k/kubeletstart.ps1", "--container-runtime=remote") ValidateKubeletArgs(ctx, s) + ValidateContainerdWindowsPriorityClass(ctx, s) ValidateCiliumIsNotRunningWindows(ctx, s) ValidateDotnetNotInstalledWindows(ctx, s) ValidateWindowsSystemServicesRestartConfiguration(ctx, s) @@ -208,6 +213,7 @@ func Test_Windows2025Gen2(t *testing.T) { ValidateWindowsDisplayVersion(ctx, s, "24H2") ValidateFileHasContent(ctx, s, "/k/kubeletstart.ps1", "--container-runtime=remote") ValidateKubeletArgs(ctx, s) + ValidateContainerdWindowsPriorityClass(ctx, s) ValidateCiliumIsNotRunningWindows(ctx, s) ValidateDotnetNotInstalledWindows(ctx, s) ValidateWindowsSystemServicesRestartConfiguration(ctx, s) @@ -235,6 +241,7 @@ func Test_Windows2025Gen2TrustedLaunch(t *testing.T) { ValidateWindowsDisplayVersion(ctx, s, "24H2") ValidateFileHasContent(ctx, s, "/k/kubeletstart.ps1", "--container-runtime=remote") ValidateKubeletArgs(ctx, s) + ValidateContainerdWindowsPriorityClass(ctx, s) ValidateCiliumIsNotRunningWindows(ctx, s) ValidateDotnetNotInstalledWindows(ctx, s) ValidateWindowsSystemServicesRestartConfiguration(ctx, s) @@ -266,6 +273,7 @@ func Test_Windows2025Gen2_WindowsCiliumNetworking(t *testing.T) { ValidateWindowsDisplayVersion(ctx, s, "24H2") ValidateFileHasContent(ctx, s, "/k/kubeletstart.ps1", "--container-runtime=remote") ValidateKubeletArgs(ctx, s) + ValidateContainerdWindowsPriorityClass(ctx, s) ValidateWindowsCiliumIsRunning(ctx, s) }, }, @@ -350,6 +358,7 @@ func Test_Windows2022_VHDCaching(t *testing.T) { ValidateWindowsDisplayVersion(ctx, s, "21H2") ValidateFileHasContent(ctx, s, "/k/kubeletstart.ps1", "--container-runtime=remote") ValidateKubeletArgs(ctx, s) + ValidateContainerdWindowsPriorityClass(ctx, s) ValidateCiliumIsNotRunningWindows(ctx, s) ValidateDotnetNotInstalledWindows(ctx, s) ValidateWindowsSystemServicesRestartConfiguration(ctx, s) @@ -378,6 +387,7 @@ func Test_Windows2025Gen2_VHDCaching(t *testing.T) { ValidateWindowsDisplayVersion(ctx, s, "24H2") ValidateFileHasContent(ctx, s, "/k/kubeletstart.ps1", "--container-runtime=remote") ValidateKubeletArgs(ctx, s) + ValidateContainerdWindowsPriorityClass(ctx, s) ValidateCiliumIsNotRunningWindows(ctx, s) ValidateDotnetNotInstalledWindows(ctx, s) ValidateWindowsSystemServicesRestartConfiguration(ctx, s) @@ -507,6 +517,7 @@ func Test_Windows2025Gen2_McrChinaCloud_Windows(t *testing.T) { ValidateWindowsDisplayVersion(ctx, s, "24H2") ValidateFileHasContent(ctx, s, "/k/kubeletstart.ps1", "--container-runtime=remote") ValidateKubeletArgs(ctx, s) + ValidateContainerdWindowsPriorityClass(ctx, s) ValidateCiliumIsNotRunningWindows(ctx, s) ValidateDotnetNotInstalledWindows(ctx, s) ValidateFileExists(ctx, s, `C:\ProgramData\containerd\certs.d\docker.io\hosts.toml`) diff --git a/e2e/shared_infra.go b/e2e/shared_infra.go index 0357613b700..cd0f5f78ac4 100644 --- a/e2e/shared_infra.go +++ b/e2e/shared_infra.go @@ -331,6 +331,7 @@ func ensureSharedBastion(ctx context.Context, rg, location string) (string, erro }, Properties: &armnetwork.BastionHostPropertiesFormat{ EnableTunneling: to.Ptr(true), + EnableIPConnect: to.Ptr(true), IPConfigurations: []*armnetwork.BastionHostIPConfiguration{ { Name: to.Ptr("bastion-ipcfg"), diff --git a/e2e/test_helpers.go b/e2e/test_helpers.go index d2df3773e63..edf75233bf4 100644 --- a/e2e/test_helpers.go +++ b/e2e/test_helpers.go @@ -85,7 +85,7 @@ func RunScenario(t *testing.T, s *Scenario) { }) return } - if scriptlessUnsupported(s) { + if config.Config.DisableScriptless || scriptlessUnsupported(s) { require.NoError(t, runScenario(t, s)) return } @@ -292,7 +292,8 @@ func prepareAKSNode(ctx context.Context, s *Scenario) (*ScenarioVM, error) { s.BootstrapConfigMutator(s.Runtime.Cluster, nbc) } if s.AKSNodeConfigMutator != nil { - nodeconfig := nbcToAKSNodeConfigV1(nbc) + nodeconfig, err := nbcToAKSNodeConfigV1(nbc) + require.NoError(s.T, err) s.AKSNodeConfigMutator(s.Runtime.Cluster, nodeconfig) s.Runtime.AKSNodeConfig = nodeconfig @@ -304,7 +305,7 @@ func prepareAKSNode(ctx context.Context, s *Scenario) (*ScenarioVM, error) { // for scriptless phase 2.5, we are using nbc cse cmd for provisioning but passing aksnodeconfig and nbc cse cmd to compare env variables // scriptless tag means provisioning with aksnodeconfig is used - if !s.Tags.Scriptless && s.BootstrapConfigMutator != nil { + if !config.Config.DisableScriptless && !s.Tags.Scriptless && s.BootstrapConfigMutator != nil { nbc.EnableScriptlessNBCCSECmd = true } } diff --git a/e2e/types.go b/e2e/types.go index 50223ad8c26..03159d694f1 100644 --- a/e2e/types.go +++ b/e2e/types.go @@ -30,6 +30,7 @@ type Tags struct { NonAnonymousACR bool GPU bool WASM bool + Kata bool BootstrapTokenFallback bool KubeletCustomConfig bool Scriptless bool @@ -207,6 +208,12 @@ type Config struct { // CustomDataWriteFiles injects additional cloud-init write_files entries into rendered customData. // This is for e2e-only validation scenarios. CustomDataWriteFiles []CustomDataWriteFile + // ForceScriptlessCompilation forces compiling and injecting the locally-built aks-node-controller + // binary even when CustomDataWriteFiles is set. By default, scenarios that use CustomDataWriteFiles + // fall back to the VHD's baked-in aks-node-controller, whose parser may predate local changes and + // cause spurious provision-config vs nbc-cmd env diffs (e.g. Corefile template changes). Set this + // when the scenario relies on local parser/baker changes being consistent. + ForceScriptlessCompilation bool // Validator is a function where the scenario can perform any extra validation checks Validator func(ctx context.Context, s *Scenario) diff --git a/e2e/validators.go b/e2e/validators.go index 3b8918c3d7c..656ace66591 100644 --- a/e2e/validators.go +++ b/e2e/validators.go @@ -250,7 +250,7 @@ func ValidateDirectoryContent(ctx context.Context, s *Scenario, path string, fil func ValidateSysctlConfig(ctx context.Context, s *Scenario, customSysctls map[string]string) { s.T.Helper() - keysToCheck := make([]string, len(customSysctls)) + keysToCheck := make([]string, 0, len(customSysctls)) for k := range customSysctls { keysToCheck = append(keysToCheck, k) } @@ -258,12 +258,106 @@ func ValidateSysctlConfig(ctx context.Context, s *Scenario, customSysctls map[st "set -ex", fmt.Sprintf("sudo sysctl %s | sed -E 's/([0-9])\\s+([0-9])/\\1 \\2/g'", strings.Join(keysToCheck, " ")), } - execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "systmctl command failed") + execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "sysctl command failed") for name, value := range customSysctls { require.Contains(s.T, execResult.stdout, fmt.Sprintf("%s = %v", name, value), "expected to find %s set to %v, but was not.\nStdout:\n%s", name, value, execResult.stdout) } } +func ValidateCustomLinuxOSConfigPersistsAfterReboot(ctx context.Context, s *Scenario, customSysctls map[string]string, customContainerdUlimits map[string]string, swapFileSizeMB int32, thpEnabled, thpDefrag string) { + s.T.Helper() + validateCustomLinuxOSConfig(ctx, s, customSysctls, customContainerdUlimits, swapFileSizeMB, thpEnabled, thpDefrag) + RebootVMAndWaitForSSH(ctx, s) + validateCustomLinuxOSConfig(ctx, s, customSysctls, customContainerdUlimits, swapFileSizeMB, thpEnabled, thpDefrag) +} + +func validateCustomLinuxOSConfig(ctx context.Context, s *Scenario, customSysctls map[string]string, customContainerdUlimits map[string]string, swapFileSizeMB int32, thpEnabled, thpDefrag string) { + s.T.Helper() + ValidateSysctlConfig(ctx, s, customSysctls) + ValidateUlimitSettings(ctx, s, customContainerdUlimits) + ValidateSwapFileConfig(ctx, s, swapFileSizeMB) + ValidateTransparentHugePageConfig(ctx, s, thpEnabled, thpDefrag) +} + +func ValidateTransparentHugePageConfig(ctx context.Context, s *Scenario, thpEnabled, thpDefrag string) { + s.T.Helper() + command := []string{"set -ex"} + if thpEnabled != "" { + command = append(command, + "cat /sys/kernel/mm/transparent_hugepage/enabled", + fmt.Sprintf("grep -Fq '[%s]' /sys/kernel/mm/transparent_hugepage/enabled", thpEnabled), + ) + } + if thpDefrag != "" { + command = append(command, + "cat /sys/kernel/mm/transparent_hugepage/defrag", + fmt.Sprintf("grep -Fq '[%s]' /sys/kernel/mm/transparent_hugepage/defrag", thpDefrag), + ) + } + + execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "transparent huge page configuration did not match expected values") +} + +func ValidateSwapFileConfig(ctx context.Context, s *Scenario, swapFileSizeMB int32) { + s.T.Helper() + if swapFileSizeMB <= 0 { + return + } + + command := []string{ + "set -ex", + "swapon --show --bytes", + "swap_file=$(awk '$3 == \"swap\" {print $1; exit}' /etc/fstab)", + "test -n \"${swap_file}\"", + "swapon --show --bytes | grep -F \"${swap_file}\"", + fmt.Sprintf("expected_bytes=$((%d * 1000 * 1000))", swapFileSizeMB), + "actual_bytes=$(stat -c %s \"${swap_file}\")", + "test \"${actual_bytes}\" -ge \"${expected_bytes}\"", + } + execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "swap file configuration did not match expected values") +} + +func RebootVMAndWaitForSSH(ctx context.Context, s *Scenario) { + s.T.Helper() + beforeRebootBootID := strings.TrimSpace(execScriptOnVMForScenarioValidateExitCode(ctx, s, "cat /proc/sys/kernel/random/boot_id", 0, "could not read boot ID before reboot").stdout) + execScriptOnVMForScenarioValidateExitCode(ctx, s, "sudo nohup sh -c 'sleep 1; systemctl reboot' >/dev/null 2>&1 &", 0, "failed to trigger VM reboot") + cleanupBastionTunnel(s.Runtime.VM.SSHClient) + s.Runtime.VM.SSHClient = nil + + waitTimeout := s.Config.WaitForSSHAfterReboot + if waitTimeout == 0 { + waitTimeout = 10 * time.Minute + } + + err := wait.PollUntilContextTimeout(ctx, 15*time.Second, waitTimeout, true, func(ctx context.Context) (bool, error) { + sshClient, err := DialSSHOverBastion(ctx, s.Runtime.Cluster.Bastion, s.Runtime.VM.PrivateIP, config.VMSSHPrivateKey) + if err != nil { + s.T.Logf("waiting for SSH after reboot: %v", err) + return false, nil + } + + bootIDCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + execResult, err := runSSHCommand(bootIDCtx, sshClient, "cat /proc/sys/kernel/random/boot_id", s.IsWindows()) + cancel() + if err != nil { + cleanupBastionTunnel(sshClient) + s.T.Logf("waiting for boot ID after reboot: %v", err) + return false, nil + } + + afterRebootBootID := strings.TrimSpace(execResult.stdout) + if afterRebootBootID == "" || afterRebootBootID == beforeRebootBootID { + cleanupBastionTunnel(sshClient) + s.T.Logf("waiting for VM reboot to complete: boot ID is still %q", afterRebootBootID) + return false, nil + } + + s.Runtime.VM.SSHClient = sshClient + return true, nil + }) + require.NoError(s.T, err, "timed out waiting for VM to reboot and accept SSH") +} + // ValidateNetworkInterfaceConfig validates network interface configuration settings using ethtool. // It identifies network interfaces with slot names matching the enP* pattern (same logic as the udev rule), // then verifies that each interface has the expected configuration settings (e.g., rx buffer size). @@ -1170,6 +1264,21 @@ func ValidateContainerd2Properties(ctx context.Context, s *Scenario, versions [] ValidateInstalledPackageVersion(ctx, s, "moby-containerd", versions[0]) + // TODO: this assertion never actually runs and always passes vacuously. + // + // execOnVMForScenarioOnUnprivilegedPod executes inside the "debugnonhost" daemonset pod, + // which runs a bare mcr.microsoft.com/cbl-mariner/base/core:2.0 image with no volume mounts + // (see daemonsetDebug in e2e/kube.go). The host filesystem is not mounted into it, so the + // containerd binary is unreachable and the command exits 127 with an empty stdout and + // "command not found" on stderr. attemptExecOnPod treats a non-zero exit as a successful + // exec, so no error surfaces, and the NotContains check below then trivially passes against + // an empty string. + // + // Two changes are needed: run this on the node via execScriptOnVMForScenarioValidateExitCode + // (as ValidateKataContainerdConfigDump in validators_kata.go now does), and check stderr as + // well as stdout, since containerd logs its warnings to stderr. Fixing it is likely to + // surface real warnings at the 11 call sites that use this validator, so it is left as a + // follow-up rather than folded into an unrelated change. execResult := execOnVMForScenarioOnUnprivilegedPod(ctx, s, "containerd config dump ") // validate containerd config dump has no warnings require.NotContains(s.T, execResult.stdout, "level=warning", "do not expect warning message when converting config file %", execResult.stdout) @@ -1428,6 +1537,27 @@ func ValidateKubeletArgs(ctx context.Context, s *Scenario) { ValidateWindowsProcessHasCliArguments(ctx, s, "kubelet.exe", []string{"--rotate-certificates=true", "--client-ca-file=c:\\k\\ca.crt", "--windows-priorityclass=ABOVE_NORMAL_PRIORITY_CLASS"}) } +// ValidateContainerdWindowsPriorityClass verifies that the containerd service is registered +// with nssm's AppPriority set to ABOVE_NORMAL_PRIORITY_CLASS, and that the running containerd +// process actually has that OS process priority class applied. +func ValidateContainerdWindowsPriorityClass(ctx context.Context, s *Scenario) { + s.T.Helper() + + nssmCommand := strings.Join([]string{ + "$ErrorActionPreference = 'Stop'", + "& \"c:\\k\\nssm.exe\" get containerd AppPriority", + }, "\n") + nssmResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, nssmCommand, 0, "could not read containerd AppPriority from nssm") + require.Equal(s.T, "ABOVE_NORMAL_PRIORITY_CLASS", strings.TrimSpace(nssmResult.stdout), "expected containerd nssm service to be configured with AppPriority=ABOVE_NORMAL_PRIORITY_CLASS") + + processCommand := strings.Join([]string{ + "$ErrorActionPreference = 'Stop'", + "(Get-Process -Name containerd -ErrorAction Stop | Select-Object -First 1).PriorityClass", + }, "\n") + processResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, processCommand, 0, "could not read containerd process priority class") + require.Equal(s.T, "AboveNormal", strings.TrimSpace(processResult.stdout), "expected containerd process to be running with AboveNormal priority class") +} + func ValidateWindowsProcessHasCliArguments(ctx context.Context, s *Scenario, processName string, arguments []string) { steps := []string{ fmt.Sprintf("(Get-CimInstance Win32_Process -Filter \"name='%[1]s'\")[0].CommandLine", processName), diff --git a/e2e/validators_kata.go b/e2e/validators_kata.go new file mode 100644 index 00000000000..44cf1920b83 --- /dev/null +++ b/e2e/validators_kata.go @@ -0,0 +1,268 @@ +package e2e + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + nodev1 "k8s.io/api/node/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // kataRuntimeHandler is the containerd runtime handler name for standard Kata Containers, + // emitted by the IsKata block of the containerd config templates in pkg/agent/baker.go. + kataRuntimeHandler = "kata" + + // kataConfigPath is the Kata configuration file referenced by the "kata" runtime handler's + // options.ConfigPath in the rendered containerd config. + kataConfigPath = "/usr/share/defaults/kata-containers/configuration.toml" + + // containerdConfigPath is where CSE / aks-node-controller writes the rendered containerd config. + containerdConfigPath = "/etc/containerd/config.toml" +) + +// kataRuntimeHandlers lists every Kata containerd runtime handler that this scenario expects to +// be configured and usable on the node. Each one is asserted in the effective containerd config +// and independently exercised by ValidateKataPodIsIsolated, so covering an additional handler is +// a one-line change here plus a call in the scenario. +// +// Note that "kata-cc" (confidential containers) is intentionally absent: its handler block is +// templated for all Kata VHDs, but it targets a different VHD than regular Kata, so this image +// cannot actually run it. +var kataRuntimeHandlers = []string{kataRuntimeHandler} + +// ValidateKataContainerdConfig asserts that AgentBaker rendered a containerd configuration +// containing the Kata runtime handlers on a Kata-enabled VHD. +// +// This is the core regression check for the IsKata blocks of the containerd config templates in +// pkg/agent/baker.go. Note that AgentPoolProfile.IsContainerdV2Distro() returns false for every +// Kata distro (pkg/agent/datamodel/types.go), so Kata nodes are always rendered from +// containerdV1ConfigTemplate / containerdV1NoGPUConfigTemplate regardless of the underlying OS. +// The assertions below therefore target the containerd 1.x plugin paths that those templates +// emit. If Kata is ever promoted to the V2 templates, this validator should fail loudly rather +// than silently pass, which is why the plugin paths are asserted explicitly. +func ValidateKataContainerdConfig(ctx context.Context, s *Scenario) { + s.T.Helper() + + require.True(s.T, s.VHD.Distro.IsKataDistro(), + "ValidateKataContainerdConfig requires a Kata distro, got %q", s.VHD.Distro) + + // The standard "kata" runtime handler, backed by the kata v2 shim. + ValidateFileHasContent(ctx, s, containerdConfigPath, `[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata]`) + ValidateFileHasContent(ctx, s, containerdConfigPath, `runtime_type = "io.containerd.kata.v2"`) + ValidateFileHasContent(ctx, s, containerdConfigPath, kataConfigPath) + + // Kata relies on snapshot annotations being forwarded to the snapshotter; the template sets + // this explicitly under IsKata and disabling it breaks image pulling for Kata pods. + ValidateFileHasContent(ctx, s, containerdConfigPath, "disable_snapshot_annotations = false") +} + +// ValidateKataContainerdConfigDump asserts that containerd itself accepted the rendered +// configuration and actually loaded the Kata runtime handlers. +// +// Checking the file alone is not enough. Kata VHDs ship their own containerd build - CSE skips +// installing one (see the "azurelinuxkata" entries in parts/common/components.json) - so the +// containerd major version on the node is decided by the image, not by AgentBaker, while the +// template AgentBaker renders is decided by the distro (IsContainerdV2Distro short-circuits to +// the v1 template for every Kata distro). The two can therefore disagree: AzureLinux V3 Kata +// currently boots containerd 2.x while being handed a containerd 1.x style config. +// +// That combination happens to work today because containerd 2.x migrates the legacy +// "io.containerd.grpc.v1.cri" runtime handlers onto the current +// "io.containerd.cri.v1.runtime" paths, but nothing guarantees it keeps doing so. This +// validator pins the property we actually care about: after containerd has parsed the config, +// the Kata handlers are present in the effective configuration and containerd raised no +// warnings while getting there. +func ValidateKataContainerdConfigDump(ctx context.Context, s *Scenario) { + s.T.Helper() + + // This must run on the node itself, not in a debug pod. The "debugnonhost" daemonset pods + // used by execOnVMForScenarioOnUnprivilegedPod run a bare CBL-Mariner base image with no + // volume mounts, so the host's containerd binary is not reachable from them and the command + // would simply exit 127. + execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, "sudo containerd config dump", 0, + "unable to dump the effective containerd config on the node") + + // The effective config is printed on stdout, but containerd logs diagnostics (including the + // "level=warning" lines we care about) on stderr, so both streams have to be inspected. + dump := execResult.stdout + diagnostics := execResult.stdout + "\n" + execResult.stderr + + // "containerd config dump" re-serializes the config and quotes TOML strings with single + // quotes, whereas the config file AgentBaker generates uses double quotes. Normalize so the + // assertions below can be written the way the config file reads. + normalizedDump := strings.ReplaceAll(dump, "'", `"`) + + // The effective config must expose every Kata runtime handler we expect. Note the trailing + // "]": without it a handler name would also match longer handlers sharing its prefix (e.g. + // "runtimes.kata" matching "runtimes.kata-preview") and pass even if the handler itself + // were missing. + for _, handler := range kataRuntimeHandlers { + assert.Contains(s.T, normalizedDump, `runtimes.`+handler+`]`, + "expected the %q runtime handler in the effective containerd config.\nDump:\n%s", handler, dump) + } + assert.Contains(s.T, normalizedDump, `runtime_type = "io.containerd.kata.v2"`, + "expected the kata v2 shim runtime_type in the effective containerd config.\nDump:\n%s", dump) + + // A warning here means containerd did not fully understand the config we generated, e.g. it + // had to fall back on deprecated handling for the legacy plugin paths the Kata templates use. + assert.NotContains(s.T, diagnostics, "level=warning", + "containerd reported warnings while parsing the AgentBaker-generated config.\nstdout:\n%s\nstderr:\n%s", + execResult.stdout, execResult.stderr) +} + +// ValidateKataHostReadiness asserts the host-side prerequisites that the Kata VHD is expected to +// ship and that the containerd config references. Without these, the containerd config would be +// syntactically valid but the kata shim would fail at pod sandbox creation time. +func ValidateKataHostReadiness(ctx context.Context, s *Scenario) { + s.T.Helper() + + // The kata shim binary that runtime_type = "io.containerd.kata.v2" resolves to. + execScriptOnVMForScenarioValidateExitCode(ctx, s, + "command -v containerd-shim-kata-v2", 0, "containerd-shim-kata-v2 is not present on the Kata VHD") + + // The Kata configuration file referenced by options.ConfigPath in the containerd config. + ValidateFileExists(ctx, s, kataConfigPath) + + // Kata VHDs deliberately opt out of automatic package updates even when unattended upgrades + // are enabled, because kata packages must be updated as a unit (including the kernel, which + // requires a reboot). See the IS_KATA branch in parts/linux/cloud-init/artifacts/cse_main.sh. + // The scenario leaves unattended upgrades enabled so this branch is genuinely exercised. + execScriptOnVMForScenarioValidateExitCode(ctx, s, + "systemctl is-enabled dnf-automatic-install.timer", 1, + "dnf-automatic-install.timer must not be enabled on Kata VHDs: kata packages have to be updated as a unit via image updates") +} + +// ValidateKataPodIsIsolated creates a RuntimeClass bound to the given Kata runtime handler, +// schedules a pod against it on the node under test, and asserts the pod is genuinely running +// inside a Kata VM. +// +// This is the end-to-end proof that the containerd config AgentBaker generated is not merely +// syntactically present but actually usable: if the runtime handler were missing or +// misconfigured, the kubelet would reject the pod with "RuntimeHandler not supported" and the +// pod would never reach Running. +// +// Isolation itself is asserted by comparing kernel releases. A Kata pod boots its own guest +// kernel, so it must report a different `uname -r` than the host; a matching value would mean +// the pod silently fell back to the shared-kernel runc runtime. +// +// The RuntimeClass is pinned to this scenario's node via Scheduling.NodeSelector so it cannot +// interfere with other scenarios running in parallel against the same cluster, and is named +// after the handler so that several handlers can be validated on one node. +func ValidateKataPodIsIsolated(ctx context.Context, s *Scenario, handler string) { + s.T.Helper() + + hostKernel := strings.TrimSpace( + execScriptOnVMForScenarioValidateExitCode(ctx, s, "uname -r", 0, "unable to read host kernel release").stdout) + require.NotEmpty(s.T, hostKernel, "host kernel release was empty") + + runtimeClassName := createKataRuntimeClass(ctx, s, handler) + pod := createKataPod(ctx, s, runtimeClassName, handler) + + execResult, err := execOnPod(ctx, s.Runtime.Kube, pod.Namespace, pod.Name, []string{"uname", "-r"}) + require.NoErrorf(s.T, err, "failed to exec in kata pod %q", pod.Name) + guestKernel := strings.TrimSpace(execResult.stdout) + require.NotEmpty(s.T, guestKernel, "kata guest kernel release was empty") + + s.T.Logf("host kernel: %q, kata guest kernel: %q", hostKernel, guestKernel) + assert.NotEqual(s.T, hostKernel, guestKernel, + "pod running under the %q RuntimeClass reported the same kernel release as the host, "+ + "which means it was not launched inside a Kata VM", handler) +} + +// createKataRuntimeClass creates a RuntimeClass for the given handler scoped to the scenario's +// node and registers its cleanup. It returns the RuntimeClass name. +func createKataRuntimeClass(ctx context.Context, s *Scenario, handler string) string { + s.T.Helper() + + kube := s.Runtime.Kube + name := truncateKataResourceName(fmt.Sprintf("%s-%s", handler, s.Runtime.VM.KubeName)) + + runtimeClass := &nodev1.RuntimeClass{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Handler: handler, + Scheduling: &nodev1.Scheduling{ + NodeSelector: map[string]string{"kubernetes.io/hostname": s.Runtime.VM.KubeName}, + }, + } + + _, err := kube.Typed.NodeV1().RuntimeClasses().Create(ctx, runtimeClass, metav1.CreateOptions{}) + require.NoErrorf(s.T, err, "failed to create RuntimeClass %q for handler %q", name, handler) + + s.T.Cleanup(func() { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + if err := kube.Typed.NodeV1().RuntimeClasses().Delete(cleanupCtx, name, metav1.DeleteOptions{}); err != nil { + s.T.Logf("could not delete RuntimeClass %s: %v", name, err) + } + }) + + return name +} + +// createKataPod creates a long-lived pod bound to the given Kata RuntimeClass on the scenario's +// node, waits for it to reach Running, and registers its cleanup. Unlike ValidatePodRunning the +// pod is kept alive after this returns so callers can exec into it. +func createKataPod(ctx context.Context, s *Scenario, runtimeClassName, handler string) *corev1.Pod { + s.T.Helper() + + kube := s.Runtime.Kube + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: truncateKataResourceName(fmt.Sprintf("%s-%s-pod", s.Runtime.VM.KubeName, handler)), + Namespace: "default", + }, + Spec: corev1.PodSpec{ + RuntimeClassName: to.Ptr(runtimeClassName), + Containers: []corev1.Container{ + { + Name: "workload", + Image: "mcr.microsoft.com/cbl-mariner/busybox:2.0", + Command: []string{"sh", "-c"}, + Args: []string{"sleep 3600"}, + }, + }, + NodeSelector: map[string]string{ + "kubernetes.io/hostname": s.Runtime.VM.KubeName, + }, + }, + } + + s.T.Logf("creating pod %q under RuntimeClass %q", pod.Name, runtimeClassName) + _, err := kube.Typed.CoreV1().Pods(pod.Namespace).Create(ctx, pod, metav1.CreateOptions{}) + require.NoErrorf(s.T, err, "failed to create kata pod %q", pod.Name) + + s.T.Cleanup(func() { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + err := kube.Typed.CoreV1().Pods(pod.Namespace).Delete(cleanupCtx, pod.Name, metav1.DeleteOptions{ + GracePeriodSeconds: to.Ptr(int64(0)), + }) + if err != nil { + s.T.Logf("could not delete pod %s: %v", pod.Name, err) + } + }) + + running, err := kube.WaitUntilPodRunning(ctx, pod.Namespace, "", "metadata.name="+pod.Name) + require.NoErrorf(s.T, err, + "kata pod %q never reached Running. This usually means containerd did not register the %q "+ + "runtime handler from the AgentBaker-generated config", pod.Name, handler) + + return running +} + +// truncateKataResourceName keeps generated Kubernetes object names within the 63 character +// DNS-1123 label limit. +func truncateKataResourceName(name string) string { + const maxLen = 63 + if len(name) <= maxLen { + return name + } + return strings.TrimRight(name[:maxLen], "-") +} diff --git a/e2e/vmss.go b/e2e/vmss.go index 5230cd14e0c..dc70b559ec5 100644 --- a/e2e/vmss.go +++ b/e2e/vmss.go @@ -245,7 +245,7 @@ func createVMSSModel(ctx context.Context, s *Scenario) armcompute.VirtualMachine customData, err = injectWriteFilesEntriesToCustomData(customData, s.Config.CustomDataWriteFiles) require.NoError(s.T, err, "failed to inject customData write_files entries") } - if !scriptlessNBCCSECmdEnabled && s.VHD.SupportsScriptless() { + if !config.Config.DisableScriptless && !scriptlessNBCCSECmdEnabled && s.VHD.SupportsScriptless() { // Validate that the custom data doesn't contain any script content, // which indicates that the scriptless CSE is working as intended decodedCustomData, err := base64.StdEncoding.DecodeString(customData) @@ -312,7 +312,13 @@ func usesScriptlessNBCCSECmd(s *Scenario) bool { } func enableScriptlessCompilation(s *Scenario) bool { - return usesScriptlessNBCCSECmd(s) && len(s.Config.CustomDataWriteFiles) <= 0 && !config.Config.DisableScriptLessCompilation && !s.Tags.NetworkIsolated && !s.VHD.Flatcar + if !usesScriptlessNBCCSECmd(s) || config.Config.DisableScriptLessCompilation || s.Tags.NetworkIsolated || s.VHD.Flatcar { + return false + } + if s.Config.ForceScriptlessCompilation { + return true + } + return len(s.Config.CustomDataWriteFiles) <= 0 } func CreateVMSSWithRetry(ctx context.Context, s *Scenario) (*ScenarioVM, error) { @@ -1197,10 +1203,7 @@ func injectWriteFilesEntriesToBoothookCustomData(decoded []byte, entries []Custo return "", err } - insertPos := strings.Index(boothookStr, "/bin/bash /opt/azure/containers/aks-node-controller-launcher.sh") - if insertPos == -1 { - insertPos = strings.Index(boothookStr, "systemctl start --no-block aks-node-controller.service") - } + insertPos := boothookInsertionPoint(boothookStr) if insertPos == -1 { return "", fmt.Errorf("cloud-boothook customData missing aks-node-controller service start") } @@ -1209,6 +1212,21 @@ func injectWriteFilesEntriesToBoothookCustomData(decoded []byte, entries []Custo return base64.StdEncoding.EncodeToString([]byte(boothookStr)), nil } +func boothookInsertionPoint(boothookStr string) int { + launcherPos := strings.Index(boothookStr, "/opt/azure/containers/aks-node-controller-launcher.sh") + if launcherPos == -1 { + launcherPos = strings.Index(boothookStr, "systemctl start --no-block aks-node-controller.service") + } + if launcherPos == -1 { + return -1 + } + lineStart := strings.LastIndex(boothookStr[:launcherPos], "\n") + if lineStart == -1 { + return 0 + } + return lineStart + 1 +} + func renderBoothookWriteFilesEntries(entries []CustomDataWriteFile) (string, error) { var entryBuilder strings.Builder entryBuilder.WriteString("\n") diff --git a/e2e/vmss_test.go b/e2e/vmss_test.go index efc7aa7a11d..dbecf3600c1 100644 --- a/e2e/vmss_test.go +++ b/e2e/vmss_test.go @@ -1,6 +1,8 @@ package e2e import ( + "encoding/base64" + "strings" "testing" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" @@ -90,3 +92,43 @@ func TestParseLinuxCSEMessageOutboundExitCode(t *testing.T) { }) } } + +func TestBoothookInsertionPoint(t *testing.T) { + tests := []struct { + name string + boothook string + wantLine string + }{ + { + name: "nohup launcher", + boothook: "#!/bin/bash\necho before\nnohup /bin/bash /opt/azure/containers/aks-node-controller-launcher.sh >/var/log/azure/aks-node-controller.output 2>&1 &\necho after\n", + wantLine: "nohup /bin/bash /opt/azure/containers/aks-node-controller-launcher.sh", + }, + { + name: "systemd launcher fallback", + boothook: "#!/bin/bash\necho before\nsystemctl start --no-block aks-node-controller.service\necho after\n", + wantLine: "systemctl start --no-block aks-node-controller.service", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pos := boothookInsertionPoint(tt.boothook) + require.GreaterOrEqual(t, pos, 0) + require.True(t, strings.HasPrefix(tt.boothook[pos:], tt.wantLine), "insertion point should be before launcher line") + }) + } +} + +func TestInjectWriteFilesEntriesToBoothookCustomDataPreservesNohupCommand(t *testing.T) { + boothook := []byte("#!/bin/bash\necho before\nnohup /bin/bash /opt/azure/containers/aks-node-controller-launcher.sh >/var/log/azure/aks-node-controller.output 2>&1 &\necho after\n") + encoded, err := injectWriteFilesEntriesToBoothookCustomData(boothook, []CustomDataWriteFile{{ + Path: "/tmp/e2e-marker", + Content: "hello", + }}) + require.NoError(t, err) + decoded, err := base64.StdEncoding.DecodeString(encoded) + require.NoError(t, err) + got := string(decoded) + require.Contains(t, got, "\nnohup /bin/bash /opt/azure/containers/aks-node-controller-launcher.sh") + require.NotContains(t, got, "nohup\n") +} diff --git a/hotfix/hotfix_generate.py b/hotfix/hotfix_generate.py index 48f58d0b72c..7cbd41877ef 100644 --- a/hotfix/hotfix_generate.py +++ b/hotfix/hotfix_generate.py @@ -86,7 +86,10 @@ "configure-azure-network.sh": "configureAzureNetworkScript", "init-aks-cloud.sh": "initAKSCloud", # Distro-specific scripts + # The updater and handler share one nodecustomdata block, so a change to + # either file hotfix-delivers both atomically. "ubuntu/ubuntu-snapshot-update.sh": "snapshotUpdateScript", + "ubuntu/security-update.sh": "securityUpdateScript", "mariner/mariner-package-update.sh": "packageUpdateScriptMariner", # Systemd services "kubelet.service": "kubeletSystemdService", @@ -97,6 +100,8 @@ "secure-tls-bootstrap.service": "secureTLSBootstrapService", "ensure-no-dup.service": "ensureNoDupEbtablesService", "measure-tls-bootstrapping-latency.service": "measureTLSBootstrappingLatencyService", + # These existing mappings remain part of the hotfix inventory, but the static + # units have no nodecustomdata write_files block and therefore are not injected. "ubuntu/snapshot-update.service": "snapshotUpdateService", "ubuntu/snapshot-update.timer": "snapshotUpdateTimer", "mariner/package-update.service": "packageUpdateServiceMariner", diff --git a/image-fetcher/main.go b/image-fetcher/main.go index 40005911584..90af856e320 100644 --- a/image-fetcher/main.go +++ b/image-fetcher/main.go @@ -24,7 +24,7 @@ const ( func main() { if len(os.Args) < 2 { fmt.Fprintf(os.Stderr, "Usage: %s [image-ref...]\n", os.Args[0]) - fmt.Fprintf(os.Stderr, "Example: %s mcr.microsoft.com/oss/kubernetes/pause:3.9\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "Example: %s mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.2\n", os.Args[0]) os.Exit(1) } diff --git a/parts/common/components.json b/parts/common/components.json index b99b0eb289b..91428eb2bfb 100644 --- a/parts/common/components.json +++ b/parts/common/components.json @@ -102,13 +102,15 @@ "multiArchVersionsV2": [ { "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes/pause", - "latestVersion": "3.6" + "latestVersion": "3.10.2", + "previousLatestVersion": "3.6" } ], "windowsVersions": [ { "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes/pause", - "latestVersion": "3.10.1" + "latestVersion": "3.10.2", + "previousLatestVersion": "3.10.1" } ] }, @@ -444,8 +446,8 @@ }, { "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes/azure-cloud-node-manager", - "latestVersion": "v1.36.4-1", - "previousLatestVersion": "v1.36.3-3" + "latestVersion": "v1.36.5-1", + "previousLatestVersion": "v1.36.4-1" } ], "windowsVersions": [ @@ -466,8 +468,8 @@ }, { "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes/azure-cloud-node-manager", - "latestVersion": "v1.36.4-windows-hpc-1", - "previousLatestVersion": "v1.36.3-windows-hpc-1" + "latestVersion": "v1.36.5-windows-hpc-1", + "previousLatestVersion": "v1.36.4-windows-hpc-1" } ] }, @@ -504,23 +506,23 @@ "multiArchVersionsV2": [ { "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes-csi/azuredisk-csi", - "latestVersion": "v1.34.4" + "latestVersion": "v1.34.5" }, { "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes-csi/azuredisk-csi", - "latestVersion": "v1.33.10" + "latestVersion": "v1.33.11" } ], "windowsVersions": [ { "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes-csi/azuredisk-csi", - "latestVersion": "v1.34.4-windows-hp", - "previousLatestVersion": "v1.34.3-windows-hp" + "latestVersion": "v1.34.5-windows-hp", + "previousLatestVersion": "v1.34.4-windows-hp" }, { "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes-csi/azuredisk-csi", - "latestVersion": "v1.33.10-windows-hp", - "previousLatestVersion": "v1.33.9-windows-hp" + "latestVersion": "v1.33.11-windows-hp", + "previousLatestVersion": "v1.33.10-windows-hp" } ] }, @@ -530,8 +532,8 @@ "multiArchVersionsV2": [ { "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes-csi/azurefile-csi", - "latestVersion": "v1.35.6", - "previousLatestVersion": "v1.35.5" + "latestVersion": "v1.35.7", + "previousLatestVersion": "v1.35.6" }, { "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes-csi/azurefile-csi", @@ -540,15 +542,15 @@ }, { "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes-csi/azurefile-csi", - "latestVersion": "v1.34.7", - "previousLatestVersion": "v1.34.6" + "latestVersion": "v1.34.8", + "previousLatestVersion": "v1.34.7" } ], "windowsVersions": [ { "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes-csi/azurefile-csi", - "latestVersion": "v1.35.6-windows-hp", - "previousLatestVersion": "v1.35.5-windows-hp" + "latestVersion": "v1.35.7-windows-hp", + "previousLatestVersion": "v1.35.6-windows-hp" }, { "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes-csi/azurefile-csi", @@ -557,8 +559,8 @@ }, { "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes-csi/azurefile-csi", - "latestVersion": "v1.34.7-windows-hp", - "previousLatestVersion": "v1.34.6-windows-hp" + "latestVersion": "v1.34.8-windows-hp", + "previousLatestVersion": "v1.34.7-windows-hp" } ] }, @@ -1041,7 +1043,7 @@ "versionsV2": [ { "renovateTag": "name=moby-containerd, repository=production, os=ubuntu, release=26.04", - "latestVersion": "2.3.2-ubuntu26.04u2" + "latestVersion": "2.3.3-ubuntu26.04u2" } ] }, @@ -1326,7 +1328,7 @@ "versionsV2": [ { "renovateTag": "name=kubernetes-cri-tools, repository=production, os=ubuntu, release=20.04", - "latestVersion": "1.34.0-ubuntu20.04u11" + "latestVersion": "1.34.0-ubuntu20.04u13" } ] }, @@ -1334,7 +1336,7 @@ "versionsV2": [ { "renovateTag": "name=kubernetes-cri-tools, repository=production, os=ubuntu, release=22.04", - "latestVersion": "1.34.0-ubuntu22.04u11" + "latestVersion": "1.34.0-ubuntu22.04u13" } ] }, @@ -1342,7 +1344,7 @@ "versionsV2": [ { "renovateTag": "name=kubernetes-cri-tools, repository=production, os=ubuntu, release=24.04", - "latestVersion": "1.34.0-ubuntu24.04u11" + "latestVersion": "1.34.0-ubuntu24.04u13" } ] }, @@ -1350,7 +1352,7 @@ "versionsV2": [ { "renovateTag": "name=kubernetes-cri-tools, repository=production, os=ubuntu, release=26.04", - "latestVersion": "1.34.0-ubuntu26.04u5" + "latestVersion": "1.34.0-ubuntu26.04u13" } ] } @@ -1380,7 +1382,7 @@ "versionsV2": [ { "renovateTag": "RPM_registry=https://packages.microsoft.com/azurelinux/3.0/prod/cloud-native/x86_64/repodata, name=kubernetes-cri-tools, os=azurelinux, release=3.0", - "latestVersion": "1.34.0-11.azl3" + "latestVersion": "1.34.0-12.azl3" } ] }, @@ -1398,7 +1400,7 @@ "versionsV2": [ { "renovateTag": "RPM_registry=https://packages.microsoft.com/azurelinux/3.0/prod/cloud-native/x86_64/repodata, name=kubernetes-cri-tools, os=azurelinux, release=3.0", - "latestVersion": "1.34.0-11.azl3" + "latestVersion": "1.34.0-12.azl3" } ] } @@ -1768,7 +1770,7 @@ "versionsV2": [ { "renovateTag": "name=ig, repository=production, os=ubuntu, release=26.04", - "latestVersion": "0.54.1-ubuntu26.04u5" + "latestVersion": "0.54.1-ubuntu26.04u6" } ] }, @@ -1776,7 +1778,7 @@ "versionsV2": [ { "renovateTag": "name=ig, repository=production, os=ubuntu, release=24.04", - "latestVersion": "0.54.1-ubuntu24.04u5" + "latestVersion": "0.54.1-ubuntu24.04u6" } ] }, @@ -1784,7 +1786,7 @@ "versionsV2": [ { "renovateTag": "name=ig, repository=production, os=ubuntu, release=22.04", - "latestVersion": "0.54.1-ubuntu22.04u5" + "latestVersion": "0.54.1-ubuntu22.04u6" } ] }, @@ -1792,7 +1794,7 @@ "versionsV2": [ { "renovateTag": "name=ig, repository=production, os=ubuntu, release=20.04", - "latestVersion": "0.54.1-ubuntu20.04u5" + "latestVersion": "0.54.1-ubuntu20.04u6" } ] } @@ -1802,7 +1804,7 @@ "versionsV2": [ { "renovateTag": "RPM_registry=https://packages.microsoft.com/azurelinux/3.0/prod/cloud-native/x86_64/repodata, name=ig, os=azurelinux, release=3.0", - "latestVersion": "0.54.1-5.azl3" + "latestVersion": "0.54.1-6.azl3" } ] } @@ -1820,8 +1822,8 @@ { "k8sVersion": "1.36", "renovateTag": "name=azure-acr-credential-provider, repository=production, os=ubuntu, release=26.04", - "latestVersion": "1.36.3-ubuntu26.04u3", - "previousLatestVersion": "1.36.2-ubuntu26.04u3" + "latestVersion": "1.36.4-ubuntu26.04u1", + "previousLatestVersion": "1.36.3-ubuntu26.04u3" } ] }, @@ -1886,20 +1888,20 @@ { "k8sVersion": "1.34", "renovateTag": "RPM_registry=https://packages.microsoft.com/azurelinux/3.0/prod/cloud-native/x86_64/repodata, name=azure-acr-credential-provider, os=azurelinux, release=3.0", - "latestVersion": "1.34.12-2.azl3", - "previousLatestVersion": "1.34.11-2.azl3" + "latestVersion": "1.34.13-1.azl3", + "previousLatestVersion": "1.34.12-2.azl3" }, { "k8sVersion": "1.35", "renovateTag": "RPM_registry=https://packages.microsoft.com/azurelinux/3.0/prod/cloud-native/x86_64/repodata, name=azure-acr-credential-provider, os=azurelinux, release=3.0", - "latestVersion": "1.35.7-2.azl3", - "previousLatestVersion": "1.35.6-2.azl3" + "latestVersion": "1.35.8-1.azl3", + "previousLatestVersion": "1.35.7-2.azl3" }, { "k8sVersion": "1.36", "renovateTag": "RPM_registry=https://packages.microsoft.com/azurelinux/3.0/prod/cloud-native/x86_64/repodata, name=azure-acr-credential-provider, os=azurelinux, release=3.0", - "latestVersion": "1.36.3-3.azl3", - "previousLatestVersion": "1.36.2-3.azl3" + "latestVersion": "1.36.4-1.azl3", + "previousLatestVersion": "1.36.3-3.azl3" } ] } @@ -2131,7 +2133,7 @@ "versionsV2": [ { "renovateTag": "name=node-exporter-kubernetes, repository=production, os=ubuntu, release=26.04", - "latestVersion": "1.9.1-ubuntu26.04u21" + "latestVersion": "1.9.1-ubuntu26.04u23" } ] }, diff --git a/parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh b/parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh index c09c023d9a5..cdbb11d85e9 100644 --- a/parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh +++ b/parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh @@ -97,6 +97,8 @@ installCriCtlPackage() { } installKubeletKubectlFromPkg() { + maskKubeletSysextUpholds || exit $ERR_K8S_INSTALL_ERR + if mergeSysexts kubelet "${2:-mcr.microsoft.com}"/oss/v2/kubernetes/kubelet-sysext "$1" \ kubectl "${2:-mcr.microsoft.com}"/oss/v2/kubernetes/kubectl-sysext "$1"; then ln -snf /usr/bin/{kubelet,kubectl} /opt/bin/ diff --git a/parts/linux/cloud-init/artifacts/cse_config.sh b/parts/linux/cloud-init/artifacts/cse_config.sh index 63a22927cc7..d0977e20026 100755 --- a/parts/linux/cloud-init/artifacts/cse_config.sh +++ b/parts/linux/cloud-init/artifacts/cse_config.sh @@ -17,16 +17,96 @@ Environment="KUBE_API_SERVER_NAME=${API_SERVER_NAME}" EOF systemctlEnableAndStart reconcile-private-hosts 30 || exit $ERR_SYSTEMCTL_START_FAIL } + configureTransparentHugePage() { - ETC_SYSFS_CONF="/etc/sysfs.conf" + local etc_sysfs_conf="/etc/sysfs.conf" + + applyTransparentHugePageValues + + if [ -n "${THP_ENABLED}" ]; then + printf 'kernel/mm/transparent_hugepage/enabled=%s\n' "${THP_ENABLED}" >> "${etc_sysfs_conf}" || exit "$ERR_SYSCTL_RELOAD" + fi + if [ -n "${THP_DEFRAG}" ]; then + printf 'kernel/mm/transparent_hugepage/defrag=%s\n' "${THP_DEFRAG}" >> "${etc_sysfs_conf}" || exit "$ERR_SYSCTL_RELOAD" + fi + reconcileTransparentHugePagePersistence +} + +applyTransparentHugePageValues() { + local thp_enabled_path="/sys/kernel/mm/transparent_hugepage/enabled" + local thp_defrag_path="/sys/kernel/mm/transparent_hugepage/defrag" + + if [ -n "${THP_ENABLED}" ]; then + printf '%s\n' "${THP_ENABLED}" > "${thp_enabled_path}" || exit "$ERR_SYSCTL_RELOAD" + fi + if [ -n "${THP_DEFRAG}" ]; then + printf '%s\n' "${THP_DEFRAG}" > "${thp_defrag_path}" || exit "$ERR_SYSCTL_RELOAD" + fi +} + +reconcileTransparentHugePagePersistence() { + if { [ -n "${THP_ENABLED}" ] || [ -n "${THP_DEFRAG}" ]; } && isMarinerOrAzureLinux "$OS" "$OS_VARIANT"; then + configureTransparentHugePageSystemdService + fi +} + +configureTransparentHugePageSystemdService() { + local service_name="aks-transparent-hugepage" + local script_path="/opt/azure/containers/aks-transparent-hugepage.sh" + local config_dir="/opt/azure/containers/aks-transparent-hugepage" + local service_path="/etc/systemd/system/${service_name}.service" + + mkdir -p "$(dirname "${script_path}")" "${config_dir}" || exit "$ERR_SYSCTL_RELOAD" if [ -n "${THP_ENABLED}" ]; then - echo "${THP_ENABLED}" > /sys/kernel/mm/transparent_hugepage/enabled - echo "kernel/mm/transparent_hugepage/enabled=${THP_ENABLED}" >> ${ETC_SYSFS_CONF} + printf '%s\n' "${THP_ENABLED}" | tee "${config_dir}/enabled" > /dev/null || exit "$ERR_SYSCTL_RELOAD" + else + rm -f "${config_dir}/enabled" || exit "$ERR_SYSCTL_RELOAD" fi if [ -n "${THP_DEFRAG}" ]; then - echo "${THP_DEFRAG}" > /sys/kernel/mm/transparent_hugepage/defrag - echo "kernel/mm/transparent_hugepage/defrag=${THP_DEFRAG}" >> ${ETC_SYSFS_CONF} + printf '%s\n' "${THP_DEFRAG}" | tee "${config_dir}/defrag" > /dev/null || exit "$ERR_SYSCTL_RELOAD" + else + rm -f "${config_dir}/defrag" || exit "$ERR_SYSCTL_RELOAD" + fi + + if ! tee "${script_path}" > /dev/null <<'EOF' +#!/bin/bash +set -e +config_dir="/opt/azure/containers/aks-transparent-hugepage" +thp_enabled_config="${config_dir}/enabled" +thp_defrag_config="${config_dir}/defrag" + +if [ -s "${thp_enabled_config}" ]; then + cat "${thp_enabled_config}" > /sys/kernel/mm/transparent_hugepage/enabled +fi +if [ -s "${thp_defrag_config}" ]; then + cat "${thp_defrag_config}" > /sys/kernel/mm/transparent_hugepage/defrag +fi +EOF + then + exit "$ERR_SYSCTL_RELOAD" + fi + chmod 0755 "${script_path}" || exit "$ERR_SYSCTL_RELOAD" + + if ! tee "${service_path}" > /dev/null </dev/null || stat -f "%Lp" "${file}" 2>/dev/null +} + +ensureSwapFileFstabEntry() { + local swap_location="$1" + local fstab_entry="${swap_location} none swap noauto,nofail 0 0" + local fstab_file="${2:-/etc/fstab}" + local fstab_dir + local fstab_mode + local temp_fstab + + fstab_dir="$(dirname "${fstab_file}")" + temp_fstab="$(mktemp "${fstab_dir}/fstab.XXXXXX")" || return 1 + fstab_mode="$(getFileMode "${fstab_file}")" || { + rm -f "${temp_fstab}" + return 1 + } + chmod "${fstab_mode}" "${temp_fstab}" || { + rm -f "${temp_fstab}" + return 1 + } + awk -v swap_location="${swap_location}" '$1 != swap_location { print }' "${fstab_file}" > "${temp_fstab}" || { + rm -f "${temp_fstab}" + return 1 + } + echo "${fstab_entry}" >> "${temp_fstab}" || { + rm -f "${temp_fstab}" + return 1 + } + mv "${temp_fstab}" "${fstab_file}" || { + rm -f "${temp_fstab}" + return 1 + } +} + +findExistingSwapFileLocation() { + local resource_disk_path + local swap_location + + if [ -L /dev/disk/azure/resource-part1 ]; then + resource_disk_path=$(findmnt -nr -o target -S "$(readlink -f /dev/disk/azure/resource-part1)" || true) + swap_location="${resource_disk_path}/swapfile" + if [ -n "${resource_disk_path}" ] && [ -f "${swap_location}" ]; then + echo "${swap_location}" + return 0 + fi + fi + + if [ -f /swapfile ]; then + echo "/swapfile" + return 0 + fi + + return 1 +} + +reconcileSwapFilePersistence() { + local swap_location="${1:-}" + + if [ -z "${swap_location}" ]; then + swap_location="$(findExistingSwapFileLocation || true)" + fi + + if [ -z "${swap_location}" ]; then + echo "No existing AKS swap file found; creating swap file for persistence reconciliation" + configureSwapFile + return 0 + fi + + ensureSwapFileFstabEntry "${swap_location}" || exit "$ERR_SWAP_CREATE_FAIL" + configureSwapFileSystemdService "${swap_location}" || exit "$ERR_SWAP_CREATE_FAIL" +} + configureSwapFile() { # https://learn.microsoft.com/en-us/troubleshoot/azure/virtual-machines/troubleshoot-device-names-problems#identify-disk-luns - swap_size_kb=$(expr ${SWAP_FILE_SIZE_MB} \* 1000) + swap_size_kb=$(expr "${SWAP_FILE_SIZE_MB}" \* 1000) swap_location="" # Attempt to use the resource disk if [ -L /dev/disk/azure/resource-part1 ]; then - resource_disk_path=$(findmnt -nr -o target -S $(readlink -f /dev/disk/azure/resource-part1)) - disk_free_kb=$(df ${resource_disk_path} | sed 1d | awk '{print $4}') - if [ "${disk_free_kb}" -gt "${swap_size_kb}" ]; then - echo "Will use resource disk for swap file" - swap_location=${resource_disk_path}/swapfile + resource_disk_path=$(findmnt -nr -o target -S "$(readlink -f /dev/disk/azure/resource-part1)" || true) + if [ -n "${resource_disk_path}" ]; then + disk_free_kb=$(df -P "${resource_disk_path}" | sed 1d | awk '{print $4}') + case "${disk_free_kb}" in + ''|*[!0-9]*) + echo "Could not determine free space on resource disk, attempting to fall back to OS disk..." + ;; + *) + if [ "${disk_free_kb}" -gt "${swap_size_kb}" ]; then + echo "Will use resource disk for swap file" + swap_location=${resource_disk_path}/swapfile + else + echo "Insufficient disk space on resource disk to create swap file: request ${swap_size_kb} free ${disk_free_kb}, attempting to fall back to OS disk..." + fi + ;; + esac else - echo "Insufficient disk space on resource disk to create swap file: request ${swap_size_kb} free ${disk_free_kb}, attempting to fall back to OS disk..." + echo "Could not determine resource disk mountpoint, attempting to fall back to OS disk..." fi fi @@ -81,12 +254,58 @@ configureSwapFile() { fi echo "Swap file will be saved to: ${swap_location}" - retrycmd_if_failure 24 5 25 fallocate -l ${swap_size_kb}K ${swap_location} || exit $ERR_SWAP_CREATE_FAIL - chmod 600 ${swap_location} - retrycmd_if_failure 24 5 25 mkswap ${swap_location} || exit $ERR_SWAP_CREATE_FAIL - retrycmd_if_failure 24 5 25 swapon ${swap_location} || exit $ERR_SWAP_CREATE_FAIL - retrycmd_if_failure 24 5 25 swapon --show | grep ${swap_location} || exit $ERR_SWAP_CREATE_FAIL - echo "${swap_location} none swap sw 0 0" >> /etc/fstab + retrycmd_if_failure 24 5 25 fallocate -l "${swap_size_kb}K" "${swap_location}" || exit "$ERR_SWAP_CREATE_FAIL" + chmod 600 "${swap_location}" + retrycmd_if_failure 24 5 25 mkswap "${swap_location}" || exit "$ERR_SWAP_CREATE_FAIL" + retrycmd_if_failure 24 5 25 swapon "${swap_location}" || exit "$ERR_SWAP_CREATE_FAIL" + swapFileIsActive "${swap_location}" || exit "$ERR_SWAP_CREATE_FAIL" + reconcileSwapFilePersistence "${swap_location}" || exit "$ERR_SWAP_CREATE_FAIL" +} + +configureSwapFileSystemdService() { + local swap_location="$1" + local service_name="aks-swapfile" + local script_path="/opt/azure/containers/aks-swapfile.sh" + local service_path="/etc/systemd/system/${service_name}.service" + local swap_mount_path + + swap_mount_path="$(dirname "${swap_location}")" + + mkdir -p "$(dirname "${script_path}")" || exit "$ERR_SWAP_CREATE_FAIL" + if ! tee "${script_path}" > /dev/null < /dev/null < /dev/null <&2 + return 1 + fi + if ! ln -sfn /dev/null "${dropinPath}"; then + echo "Failed to mask kubelet sysext systemd drop-in ${dropinPath}" >&2 + return 1 + fi +} + isARM64() { if [ "$(getCPUArch)" = "arm64" ]; then echo 1 diff --git a/parts/linux/cloud-init/artifacts/cse_main.sh b/parts/linux/cloud-init/artifacts/cse_main.sh index 0ff1fd16150..5ad1fb5f9c1 100755 --- a/parts/linux/cloud-init/artifacts/cse_main.sh +++ b/parts/linux/cloud-init/artifacts/cse_main.sh @@ -425,6 +425,15 @@ function nodePrep { logs_to_events "AKS.CSE.fetch_and_cache_imds_instance_metadata" fetch_and_cache_imds_instance_metadata reconcileVulnerableKernelModuleMitigation + if [ "${SHOULD_CONFIG_TRANSPARENT_HUGE_PAGE}" = "true" ]; then + logs_to_events "AKS.CSE.applyTransparentHugePageValues" applyTransparentHugePageValues + logs_to_events "AKS.CSE.reconcileTransparentHugePagePersistence" reconcileTransparentHugePagePersistence + fi + + if [ "${SHOULD_CONFIG_SWAP_FILE}" = "true" ]; then + logs_to_events "AKS.CSE.reconcileSwapFilePersistence" reconcileSwapFilePersistence + fi + # IMPORTANT NOTE: We do this here since this function can mutate kubelet flags and node labels, # which is used by configureK8s and other functions. Thus, we need to make sure flag and label content is correct beforehand. logs_to_events "AKS.CSE.configureKubeletServing" configureKubeletServing diff --git a/parts/linux/cloud-init/artifacts/localdns.sh b/parts/linux/cloud-init/artifacts/localdns.sh index 6f07c1ee455..0156e276201 100644 --- a/parts/linux/cloud-init/artifacts/localdns.sh +++ b/parts/linux/cloud-init/artifacts/localdns.sh @@ -24,6 +24,8 @@ LOCALDNS_CGROUP_DIR="/sys/fs/cgroup/localdns.slice/localdns.service" LOCALDNS_CORE_FILE="${LOCALDNS_SCRIPT_PATH}/localdns.corefile" # This is the localdns corefile that has updated UpstreamDNSServerIPs and will be used by the localdns systemd unit. +LIVEPATCHED_LOCALDNS_CORE_FILE="${LOCALDNS_SCRIPT_PATH}/livepatched.localdns.corefile" + UPDATED_LOCALDNS_CORE_FILE="${LOCALDNS_SCRIPT_PATH}/updated.localdns.corefile" # This is slice file used by localdns systemd unit. @@ -69,6 +71,10 @@ START_LOCALDNS_TIMEOUT=10 DNS_HEALTH_CHECK_TIMEOUT=2 DNS_HEALTH_CHECK_TRIES=2 +AKS_NODE_CONTROLLER_BINARY="/opt/azure/containers/aks-node-controller" +LOCALDNS_LIVE_PATCHING_COMPONENT_NAME="localDNS" +LOCALDNS_LIVE_PATCHING_STATUS_ANNOTATION="kubernetes.azure.com/live-patching-status" + # Function definitions used in this file. # functions defined until "${__SOURCED__:+return}" are sourced and tested in - # spec/parts/linux/cloud-init/artifacts/localdns_spec.sh. @@ -157,6 +163,14 @@ regenerate_localdns_corefile() { return 0 } +localdns_source_corefile() { + if [ -s "${LIVEPATCHED_LOCALDNS_CORE_FILE:-}" ]; then + echo "${LIVEPATCHED_LOCALDNS_CORE_FILE}" + return 0 + fi + echo "${LOCALDNS_CORE_FILE}" +} + # Replace AzureDNSIP in corefile with VNET DNS ServerIPs if necessary. replace_azurednsip_in_corefile() { if [ -z "${RESOLV_CONF:-}" ]; then @@ -187,8 +201,10 @@ replace_azurednsip_in_corefile() { # and also not equal to the localdns node listener IP to avoid creating a circular dependency. # Corefile will have 168.63.129.16 when user input has VnetDNS value for forwarddestination. # Note - For root domain under VnetDNSOverrides, all DNS traffic should be forwarded to VnetDNS. - cp "${LOCALDNS_CORE_FILE}" "${UPDATED_LOCALDNS_CORE_FILE}" || { - echo "Failed to copy ${LOCALDNS_CORE_FILE} to ${UPDATED_LOCALDNS_CORE_FILE}" + local source_corefile + source_corefile="$(localdns_source_corefile)" + cp "${source_corefile}" "${UPDATED_LOCALDNS_CORE_FILE}" || { + echo "Failed to copy ${source_corefile} to ${UPDATED_LOCALDNS_CORE_FILE}" return 1 } @@ -304,6 +320,119 @@ replace_azurednsip_in_corefile() { return 0 } +refresh_localdns_corefile_from_lps() { + if [ -z "${LIVEPATCHED_LOCALDNS_CORE_FILE:-}" ]; then + echo "LIVEPATCHED_LOCALDNS_CORE_FILE is not set or is empty." + return 1 + fi + + if [ ! -x "${AKS_NODE_CONTROLLER_BINARY}" ]; then + echo "AKS node controller binary not found at ${AKS_NODE_CONTROLLER_BINARY}; skipping LocalDNS LPS config fetch." + return 0 + fi + + # Write the LPS-provided Corefile to the livepatched source file; VNET DNS replacement + # later derives UPDATED_LOCALDNS_CORE_FILE from this file before CoreDNS starts. + if "${AKS_NODE_CONTROLLER_BINARY}" fetch-localdns-config --output "${LIVEPATCHED_LOCALDNS_CORE_FILE}"; then + echo "Completed LocalDNS LPS config fetch." + return 0 + fi + + echo "LocalDNS LPS config fetch failed; continuing with existing corefile." + return 0 +} + +localdns_corefile_version_file() { + echo "${LIVEPATCHED_LOCALDNS_CORE_FILE}.version" +} + +wait_for_kubeconfig_and_node() { + if [ ! -x /opt/bin/kubectl ]; then + echo "kubectl binary not found at /opt/bin/kubectl, skipping annotation." >&2 + return 1 + fi + + local kubeconfig="${KUBECONFIG:-/var/lib/kubelet/kubeconfig}" + local wait_count=0 + local max_wait="${KUBECONFIG_WAIT_ATTEMPTS:-60}" + while [ ! -f "${kubeconfig}" ]; do + if [ $wait_count -ge $max_wait ]; then + echo "Timeout waiting for kubeconfig at ${kubeconfig} after ${max_wait} attempts, skipping annotation." >&2 + return 1 + fi + echo "Waiting for TLS bootstrapping to complete (attempt $((wait_count + 1))/${max_wait})..." >&2 + sleep 3 + wait_count=$((wait_count + 1)) + done + echo "Kubeconfig found at ${kubeconfig}" >&2 + + local node_name + node_name=$(hostname) + if [ -z "${node_name}" ]; then + echo "Cannot get node name, skipping annotation." >&2 + return 1 + fi + node_name=$(echo "$node_name" | tr '[:upper:]' '[:lower:]') + + echo "Waiting for node ${node_name} to be registered in the cluster..." >&2 + local node_wait_count=0 + local max_node_wait="${NODE_REGISTRATION_WAIT_ATTEMPTS:-30}" + while [ $node_wait_count -lt $max_node_wait ]; do + if /opt/bin/kubectl --kubeconfig "${kubeconfig}" get node "${node_name}" >/dev/null 2>&1; then + echo "${kubeconfig}|${node_name}" + return 0 + fi + echo "Waiting for node registration (attempt $((node_wait_count + 1))/${max_node_wait})..." >&2 + sleep 3 + node_wait_count=$((node_wait_count + 1)) + done + + echo "Timeout waiting for node ${node_name} to be registered after ${max_node_wait} attempts, skipping annotation." >&2 + return 1 +} + +annotate_node_with_localdns_livepatch_status() { + local version_file + version_file="$(localdns_corefile_version_file)" + if [ ! -s "${version_file}" ]; then + echo "LocalDNS corefile version file not found at ${version_file}, skipping live patching status annotation." + return 0 + fi + + local corefile_version + corefile_version="$(tr -d '[:space:]' < "${version_file}")" + if [ -z "${corefile_version}" ]; then + echo "LocalDNS corefile version file is empty, skipping live patching status annotation." + return 0 + fi + + local kube_node + kube_node="$(wait_for_kubeconfig_and_node)" || return 0 + local kubeconfig="${kube_node%%|*}" + local node_name="${kube_node#*|}" + local current_status + current_status=$(/opt/bin/kubectl --kubeconfig "${kubeconfig}" get node "${node_name}" -o "jsonpath={.metadata.annotations['kubernetes\.azure\.com/live-patching-status']}" 2>/dev/null || true) + if [ -z "${current_status}" ]; then + current_status='{}' + fi + + local updated_status + if ! updated_status="$(printf '%s' "${current_status}" | jq -c \ + --arg component "${LOCALDNS_LIVE_PATCHING_COMPONENT_NAME}" \ + --arg current "${corefile_version}" \ + '.components = (.components // {}) | .components[$component].current = $current')"; then + echo "Failed to render LocalDNS live patching status annotation." + return 0 + fi + + echo "Setting LocalDNS live patching current version ${corefile_version} for node ${node_name}." + if /opt/bin/kubectl --kubeconfig "${kubeconfig}" annotate --overwrite node "${node_name}" "${LOCALDNS_LIVE_PATCHING_STATUS_ANNOTATION}=${updated_status}"; then + echo "Successfully set LocalDNS live patching status annotation." + else + echo "Warning: Failed to set LocalDNS live patching status annotation (this is non-fatal)." + fi +} + # Build iptables rules to skip conntrack for DNS traffic to localdns. build_localdns_iptable_rules() { # These rules skip conntrack for DNS traffic to the local DNS service IPs to save conntrack table space. @@ -1026,6 +1155,12 @@ if ! wait_for_localdns_removed_from_resolv_conf 5; then exit $ERR_LOCALDNS_FAIL fi +# Fetch LocalDNS config from LPS if present. This is fail-open: no config or fetch errors keep the +# locally generated corefile. If LPS returns a usable profile/corefile, LIVEPATCHED_LOCALDNS_CORE_FILE +# is written before VNET DNS replacement builds UPDATED_LOCALDNS_CORE_FILE for CoreDNS. +# --------------------------------------------------------------------------------------------------------------------- +refresh_localdns_corefile_from_lps + # Replace AzureDNSIP in corefile with VNET DNS ServerIPs. # --------------------------------------------------------------------------------------------------------------------- replace_azurednsip_in_corefile || exit $ERR_LOCALDNS_FAIL @@ -1073,6 +1208,17 @@ echo "Startup complete - serving node and pod DNS traffic." # Export initial resource metrics so the exporter has data before the first watchdog tick. export_resource_metrics +# The generic knead live-patching loop owns kubernetes.azure.com/live-patching-status at runtime. +# Keep this legacy/bootstrap writer opt-in to avoid racing knead's status update. +# -------------------------------------------------------------------------------------------------------------------- +if [ "${LOCALDNS_ENABLE_LEGACY_LIVEPATCH_STATUS:-false}" = "true" ]; then + annotate_node_with_localdns_livepatch_status & + LOCALDNS_LIVEPATCH_ANNOTATION_PID=$! + echo "Started LocalDNS live-patching status annotation in background (PID: ${LOCALDNS_LIVEPATCH_ANNOTATION_PID})" +else + echo "Skipping LocalDNS live-patching status annotation; knead owns live-patching-status." +fi + # Set node annotation to indicate hosts plugin is in use (if applicable). # -------------------------------------------------------------------------------------------------------------------- # Only run when hosts plugin is currently enabled or was previously enabled (marker exists). diff --git a/parts/linux/cloud-init/artifacts/ubuntu/gb/containerd-nvidia.toml b/parts/linux/cloud-init/artifacts/ubuntu/gb/containerd-nvidia.toml index 88aa0fa0222..4f3771ab9a8 100644 --- a/parts/linux/cloud-init/artifacts/ubuntu/gb/containerd-nvidia.toml +++ b/parts/linux/cloud-init/artifacts/ubuntu/gb/containerd-nvidia.toml @@ -7,7 +7,7 @@ version = 2 [plugins] [plugins."io.containerd.grpc.v1.cri"] - sandbox_image = "mcr.microsoft.com/oss/kubernetes/pause:3.6" + sandbox_image = "mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.2" [plugins."io.containerd.grpc.v1.cri".containerd] default_runtime_name = "nvidia" diff --git a/parts/linux/cloud-init/artifacts/ubuntu/security-update.sh b/parts/linux/cloud-init/artifacts/ubuntu/security-update.sh new file mode 100644 index 00000000000..7b451e300f8 --- /dev/null +++ b/parts/linux/cloud-init/artifacts/ubuntu/security-update.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash + +# Applies the securityPatch component selected for this node's agent pool. This +# preserves the legacy Ubuntu snapshot-update behavior while moving desired state into the +# generic live-patching ConfigMap and status contract owned by the updater loop. + +: "${SECURITY_PATCH_CONFIG_DIR:=/var/lib/security-patch}" +: "${SECURITY_PATCH_DEFAULT_ENDPOINT:=snapshot.ubuntu.com}" + +security_patch_unattended_upgrade() { + local attempt + + for attempt in $(seq 1 10); do + if unattended-upgrade -v; then + echo "executed unattended upgrade ${attempt} times" + return 0 + fi + if [ "${attempt}" -lt 10 ]; then + sleep 5 + fi + done + + return 1 +} + +security_patch_generate_sources_list() { + local endpoint="$1" + local golden_timestamp="$2" + local code_name="$3" + + mkdir -p "${SECURITY_PATCH_CONFIG_DIR}" || return 1 + if [ "${endpoint}" = "${SECURITY_PATCH_DEFAULT_ENDPOINT}" ]; then + cat < "${SECURITY_PATCH_CONFIG_DIR}/sources.list" || return 1 +deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name} main restricted +deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-updates main restricted +deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name} universe +deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-updates universe +deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name} multiverse +deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-updates multiverse +deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-backports main restricted universe multiverse +deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-security main restricted +deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-security universe +deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-security multiverse +EOF + else + cat < "${SECURITY_PATCH_CONFIG_DIR}/sources.list" || return 1 +deb http://${endpoint}/ubuntu ${code_name} main restricted +deb http://${endpoint}/ubuntu ${code_name}-updates main restricted +deb http://${endpoint}/ubuntu ${code_name} universe +deb http://${endpoint}/ubuntu ${code_name}-updates universe +deb http://${endpoint}/ubuntu ${code_name} multiverse +deb http://${endpoint}/ubuntu ${code_name}-updates multiverse +deb http://${endpoint}/ubuntu ${code_name}-backports main restricted universe multiverse +deb http://${endpoint}/ubuntu ${code_name}-security main restricted +deb http://${endpoint}/ubuntu ${code_name}-security universe +deb http://${endpoint}/ubuntu ${code_name}-security multiverse +EOF + fi + + cat < "${SECURITY_PATCH_CONFIG_DIR}/apt.conf" || return 1 +Dir::Etc::sourcelist "${SECURITY_PATCH_CONFIG_DIR}/sources.list"; +Dir::Etc::sourceparts ""; +EOF +} + +security_patch_repo_endpoint() { + local node_json="$1" + local repo_service + local private_ip_regex='^((10\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})|(172\.(1[6-9]|2[0-9]|3[01])\.[0-9]{1,3}\.[0-9]{1,3})|(192\.168\.[0-9]{1,3}\.[0-9]{1,3}))$' + + if ! repo_service="$(printf '%s' "${node_json}" | jq -r '.metadata.annotations["kubernetes.azure.com/live-patching-repo-service"] // empty')"; then + echo "failed to read live patching repo service annotation" + return 1 + fi + + if [ -z "${repo_service}" ]; then + echo "${SECURITY_PATCH_DEFAULT_ENDPOINT}" + elif printf '%s' "${repo_service}" | grep -Eq "${private_ip_regex}"; then + echo "${repo_service}" + else + echo "ignoring invalid live patching repo service: ${repo_service}" >&2 + echo "${SECURITY_PATCH_DEFAULT_ENDPOINT}" + fi +} + +# Returns success when the desired security patch timestamp for this node's +# agent pool matches the timestamp in the locally checkpointed component state. +securityPatchIsCurrent() { + local desired_payload="$1" + local current_payload="$2" + local node_json="$3" + local agent_pool + local desired_timestamp + local current_timestamp + + if ! agent_pool="$(printf '%s' "${node_json}" | jq -er '.metadata.labels["kubernetes.azure.com/agentpool"] // empty')"; then + return 1 + fi + if ! desired_timestamp="$(printf '%s' "${desired_payload}" | jq -er --arg agentPool "${agent_pool}" '.agentPools[$agentPool].goldenTimestamp // empty')"; then + return 1 + fi + if ! current_timestamp="$(printf '%s' "${current_payload}" | jq -er --arg agentPool "${agent_pool}" '.agentPools[$agentPool].goldenTimestamp // empty')"; then + return 1 + fi + + [ "${desired_timestamp}" = "${current_timestamp}" ] +} + +# Applies the security patch timestamp selected for this node's agent pool and, +# after patching succeeds, updates the legacy annotation consumed by the RP. +updateSecurityPatch() { + local component_payload="${1:-}" + local node_json="${2:-}" + local agent_pool + local golden_timestamp + local node_name + local repo_endpoint + local code_name + local apt_opts="-o Acquire::http::Timeout=300 -o Acquire::https::Timeout=300 -o Acquire::Retries=3" + + if [ -z "${node_json}" ]; then + echo "node JSON is required" + return 1 + fi + if ! agent_pool="$(printf '%s' "${node_json}" | jq -r '.metadata.labels["kubernetes.azure.com/agentpool"] // empty')"; then + echo "failed to read node agent pool label" + return 1 + fi + if [ -z "${agent_pool}" ]; then + echo "node agent pool label is not set" + return 1 + fi + if ! node_name="$(printf '%s' "${node_json}" | jq -er '.metadata.name // empty')"; then + echo "node name is not set" + return 1 + fi + + if ! golden_timestamp="$(printf '%s' "${component_payload}" | jq -er --arg agentPool "${agent_pool}" '.agentPools[$agentPool].goldenTimestamp // empty')"; then + echo "securityPatch profile is missing for agent pool: ${agent_pool}" + return 1 + fi + if ! printf '%s' "${golden_timestamp}" | grep -Eq '^[0-9]{8}T[0-9]{6}Z$'; then + echo "securityPatch goldenTimestamp is invalid: ${golden_timestamp}" + return 1 + fi + + if ! repo_endpoint="$(security_patch_repo_endpoint "${node_json}")"; then + return 1 + fi + if ! code_name="$(lsb_release -cs)" || [ -z "${code_name}" ]; then + echo "failed to determine Ubuntu codename" + return 1 + fi + if ! security_patch_generate_sources_list "${repo_endpoint}" "${golden_timestamp}" "${code_name}"; then + echo "failed to generate securityPatch apt configuration" + return 1 + fi + + export APT_CONFIG="${SECURITY_PATCH_CONFIG_DIR}/apt.conf" + if ! apt_get_update_with_opts "${apt_opts}"; then + echo "apt_get_update_with_opts failed" + return 1 + fi + if ! security_patch_unattended_upgrade; then + echo "unattended_upgrade failed" + return 1 + fi + + # Keep the legacy RP status channel current during migration to live-patching-status. + # shellcheck disable=SC2086 + if ! $KUBECTL annotate --overwrite node "${node_name}" "kubernetes.azure.com/live-patching-current-timestamp=${golden_timestamp}"; then + echo "failed to update legacy securityPatch status annotation" + return 1 + fi + + echo "securityPatch update completed successfully: ${golden_timestamp}" +} + +${__SOURCED__:+return} + +# shellcheck disable=SC1091 +source /opt/azure/containers/provision_source_distro.sh \ No newline at end of file diff --git a/parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh b/parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh index 6e200a8fb8b..b631a342ab3 100755 --- a/parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh +++ b/parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh @@ -3,151 +3,422 @@ set -o nounset set -e -# Global constants used in this file. -# ------------------------------------------------------------------------------------------------- -SECURITY_PATCH_CONFIG_DIR=/var/lib/security-patch -KUBECONFIG="/var/lib/kubelet/kubeconfig" -KUBECTL="/opt/bin/kubectl --kubeconfig ${KUBECONFIG}" -DEFAULT_ENDPOINT="snapshot.ubuntu.com" - -# Function definitions used in this file. -# functions defined until "${__SOURCED__:+return}" are sourced and tested in - -# spec/parts/linux/cloud-init/artifacts/ubuntu-snapshot-update_spec.sh. -# ------------------------------------------------------------------------------------------------- -# Execute unattended-upgrade -unattended_upgrade() { - retries=10 - for i in $(seq 1 $retries); do - unattended-upgrade -v && break - if [ $i -eq $retries ]; then - return 1 - else sleep 5 - fi - done - echo Executed unattended upgrade $i times -} +# Runs the generic node-side component reconciliation loop. Component handlers own +# their payload schema and node mutations; this loop owns annotations, ConfigMap +# validation, dispatch, local component state, and final success reporting. -generate_sources_list() { - local endpoint="$1" - local golden_timestamp="$2" - local code_name="$3" - - mkdir -p "${SECURITY_PATCH_CONFIG_DIR}" - if [ "${endpoint}" = "${DEFAULT_ENDPOINT}" ]; then - cat << EOF > "${SECURITY_PATCH_CONFIG_DIR}/sources.list" -deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name} main restricted -deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-updates main restricted -deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name} universe -deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-updates universe -deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name} multiverse -deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-updates multiverse -deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-backports main restricted universe multiverse -deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-security main restricted -deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-security universe -deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-security multiverse -EOF - else - cat << EOF > "${SECURITY_PATCH_CONFIG_DIR}/sources.list" -deb http://${endpoint}/ubuntu ${code_name} main restricted -deb http://${endpoint}/ubuntu ${code_name}-updates main restricted -deb http://${endpoint}/ubuntu ${code_name} universe -deb http://${endpoint}/ubuntu ${code_name}-updates universe -deb http://${endpoint}/ubuntu ${code_name} multiverse -deb http://${endpoint}/ubuntu ${code_name}-updates multiverse -deb http://${endpoint}/ubuntu ${code_name}-backports main restricted universe multiverse -deb http://${endpoint}/ubuntu ${code_name}-security main restricted -deb http://${endpoint}/ubuntu ${code_name}-security universe -deb http://${endpoint}/ubuntu ${code_name}-security multiverse -EOF - fi - - cat << EOF > "${SECURITY_PATCH_CONFIG_DIR}/apt.conf" -Dir::Etc::sourcelist "${SECURITY_PATCH_CONFIG_DIR}/sources.list"; -Dir::Etc::sourceparts ""; -EOF - - echo "live patching configuration generated successfully" -} +: "${KUBECONFIG:=/var/lib/kubelet/kubeconfig}" +: "${KUBECTL:=/opt/bin/kubectl --kubeconfig ${KUBECONFIG}}" +: "${KNEAD_COMPONENT_CONFIG_NAMESPACE:=kube-system}" +: "${KNEAD_COMPONENT_CONFIGMAP:=live-patching-config}" +: "${KNEAD_COMPONENT_CONFIG_KEY_JSONPATH:=live-patching-config\.json}" +: "${KNEAD_COMPONENT_GOAL_ANNOTATION:=kubernetes.azure.com/live-patching-config-goal-hash}" +: "${KNEAD_COMPONENT_STATUS_ANNOTATION:=kubernetes.azure.com/live-patching-status}" +: "${KNEAD_COMPONENT_STATE_FILE:=/var/lib/aks/live-patching/current.json}" +: "${KNEAD_KUBECONFIG_WAIT_TIMEOUT_SECONDS:=600}" + +KNEAD_COMPONENT_RESULTS='{}' +KNEAD_COMPONENT_RESULTS_VALID=true + +# Waits for kubelet credentials so kubectl can read Node and ConfigMap state. +knead_wait_for_kubeconfig() { + local wait_started_at="${SECONDS}" -main() { - # At startup, we need to wait for kubelet to finish TLS bootstrapping to create the kubeconfig file. - while [ ! -f ${KUBECONFIG} ]; do - echo 'Waiting for TLS bootstrapping' + while [ ! -f "${KUBECONFIG}" ]; do + if [ $((SECONDS - wait_started_at)) -ge "${KNEAD_KUBECONFIG_WAIT_TIMEOUT_SECONDS}" ]; then + echo "timed out waiting for kubelet kubeconfig after ${KNEAD_KUBECONFIG_WAIT_TIMEOUT_SECONDS}s" >&2 + return 1 + fi + echo "waiting for kubelet kubeconfig" sleep 3 done +} + +# Reads this Node using the lowercase hostname expected from cloud provider registration. +knead_read_node() { + local node_name - node_name=$(hostname) + node_name="$(hostname)" if [ -z "${node_name}" ]; then - echo "cannot get node name" - exit 1 + echo "cannot get node name" >&2 + return 1 + fi + + node_name="$(printf '%s' "${node_name}" | tr '[:upper:]' '[:lower:]')" + + # shellcheck disable=SC2086 + $KUBECTL get node "${node_name}" -o json +} + +# Returns the value of the given annotation. +knead_get_node_annotation() { + local node_json="$1" + local annotation="$2" + + printf '%s' "${node_json}" | jq -r --arg annotation "${annotation}" '.metadata.annotations[$annotation] // empty' +} + +# Reads and validates the generic ConfigMap contract before any component handler runs. +# +# Knead validates the component envelope. Each handler owns its decoded +# nodeConfig schema. The goal must be the bare sha256 digest of the exact +# ConfigMap value read by this node. +knead_read_configmap() { + local goal="$1" + local payload + local payload_hash + + # shellcheck disable=SC2086 + if ! payload="$($KUBECTL get cm -n "${KNEAD_COMPONENT_CONFIG_NAMESPACE}" "${KNEAD_COMPONENT_CONFIGMAP}" -o "jsonpath={.data.${KNEAD_COMPONENT_CONFIG_KEY_JSONPATH}}")"; then + echo "failed to read live-patching-config ConfigMap" >&2 + return 1 fi - # Azure cloud provider assigns node name as the lowner case of the hostname - node_name=$(echo "$node_name" | tr '[:upper:]' '[:lower:]') + if ! printf '%s' "${payload}" | jq -e ' + (.components | type == "array") and + (.components | all( + (.name | type == "string") and + (.name | length > 0) and + (.nodeConfig | type == "string") + )) and + ([.components[].name] | length) == ([.components[].name] | unique | length) + ' > /dev/null; then + echo "live-patching-config payload has invalid envelope" >&2 + return 1 + fi - # retrieve golden timestamp from node annotation - golden_timestamp=$($KUBECTL get node ${node_name} -o jsonpath="{.metadata.annotations['kubernetes\.azure\.com/live-patching-golden-timestamp']}") - if [ -z "${golden_timestamp}" ]; then - echo "golden timestamp is not set, skip live patching" - exit 0 + if ! printf '%s' "${goal}" | grep -Eq '^[0-9a-f]{64}$'; then + echo "live-patching goal hash must be a 64-character lowercase sha256 digest" >&2 + return 1 fi - echo "golden timestamp is: ${golden_timestamp}" - current_timestamp=$($KUBECTL get node ${node_name} -o jsonpath="{.metadata.annotations['kubernetes\.azure\.com/live-patching-current-timestamp']}") - if [ -n "${current_timestamp}" ]; then - echo "current timestamp is: ${current_timestamp}" + payload_hash="$(printf '%s' "${payload}" | sha256sum | awk '{print $1}')" + if [ "${payload_hash}" != "${goal}" ]; then + echo "live-patching goal hash does not match ConfigMap payload: goal=${goal}, payload=${payload_hash}" >&2 + return 1 + fi - if [ "${golden_timestamp}" = "${current_timestamp}" ]; then - echo "golden and current timestamp is the same, nothing to patch" - exit 0 + printf '%s' "${payload}" +} + +# Returns success when this exact component config was applied successfully but +# the overall Node status did not converge, such as when the annotation update or +# a sibling component failed. This local checkpoint prevents repeating successful +# work on the next retry. Missing or malformed state is treated as not current. +knead_component_is_current() { + local component="$1" + local component_payload="$2" + local node_json="$3" + local component_comparator="$4" + local current_payload + + if [ ! -f "${KNEAD_COMPONENT_STATE_FILE}" ]; then + return 1 + fi + + if ! current_payload="$(jq -er --arg component "${component}" \ + '.components | map(select(.name == $component)) | last | .nodeConfig' \ + "${KNEAD_COMPONENT_STATE_FILE}" 2> /dev/null)"; then + return 1 + fi + + "${component_comparator}" "${component_payload}" "${current_payload}" "${node_json}" +} + +# Records one successfully applied component without changing sibling +# state. Missing or malformed state is rebuilt from an empty component list. +knead_write_component_state() { + local component="$1" + local component_payload="$2" + local state='{"components":[]}' + local state_dir + local state_tmp + local updated_at + + state_dir="$(dirname "${KNEAD_COMPONENT_STATE_FILE}")" + state_tmp="${KNEAD_COMPONENT_STATE_FILE}.tmp" + updated_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + + if ! mkdir -p "${state_dir}"; then + echo "failed to create knead-component state directory" + return 1 + fi + if [ -f "${KNEAD_COMPONENT_STATE_FILE}" ]; then + if ! state="$(jq -c 'select(.components | type == "array") | {components: .components}' "${KNEAD_COMPONENT_STATE_FILE}" 2> /dev/null)" || [ -z "${state}" ]; then + state='{"components":[]}' fi fi + if ! printf '%s' "${state}" | jq \ + --arg component "${component}" \ + --arg componentPayload "${component_payload}" \ + --arg updatedAt "${updated_at}" \ + '.updatedAt = $updatedAt | .components = ([.components[] | select(.name != $component)] + [{name: $component, nodeConfig: $componentPayload}])' \ + > "${state_tmp}"; then + echo "failed to render knead-component state" + return 1 + fi + if ! mv "${state_tmp}" "${KNEAD_COMPONENT_STATE_FILE}"; then + echo "failed to write knead-component state" + return 1 + fi +} + +# Adds a component result to the status collection. +knead_set_component_result() { + local component="$1" + local code="$2" + local updated_results - # Network isolated cluster can't access the internet, so we deploy a live patching repo service in the cluster - # The node will use the live patching repo service to download the repo metadata and packages - # If the annotation is not set, we will use the ubuntu snapshot repo - live_patching_repo_service=$($KUBECTL get node ${node_name} -o jsonpath="{.metadata.annotations['kubernetes\.azure\.com/live-patching-repo-service']}") - # Limit the live patching repo service to private IPs in the range of 10.x.x.x, 172.16.x.x - 172.31.x.x, and 192.168.x.x - private_ip_regex="^((10\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})|(172\.(1[6-9]|2[0-9]|3[01])\.[0-9]{1,3}\.[0-9]{1,3})|(192\.168\.[0-9]{1,3}\.[0-9]{1,3}))$" - # shellcheck disable=SC3010 - if [ -n "${live_patching_repo_service}" ] && [[ ! "${live_patching_repo_service}" =~ $private_ip_regex ]]; then - echo "Ignore invalid live patching repo service: ${live_patching_repo_service}" - live_patching_repo_service="" + if ! updated_results="$(printf '%s' "${KNEAD_COMPONENT_RESULTS}" | jq -ce \ + --arg component "${component}" \ + --arg code "${code}" \ + '.[$component] = {code: $code}')" || [ -z "${updated_results}" ]; then + echo "failed to record component result: ${component}=${code}" + KNEAD_COMPONENT_RESULTS_VALID=false + return 1 fi - repo_endpoint="${DEFAULT_ENDPOINT}" - if [ -z "${live_patching_repo_service}" ]; then - echo "live patching repo service is not set, use ubuntu snapshot repo" - else - echo "live patching repo service is: ${live_patching_repo_service}" - repo_endpoint="${live_patching_repo_service}" + KNEAD_COMPONENT_RESULTS="${updated_results}" +} + +# Dispatches every known component and returns failure only after all have been attempted. +# +# This is the main error boundary for node disruption. A broken component should +# not stop unrelated handlers from making progress, so this function records +# failures and continues dispatching. Unknown components are skipped so older +# VHDs can still converge when a newer component appears in the shared config. +# +# Keep component payload parsing behind each handler. The generic loop should know +# only which handler owns a component name, not the shape of that component's JSON. +knead_apply_components() { + local payload="$1" + local node_json="$2" + local failed_components="" + local component + local component_count + local component_comparator + local component_handler + local component_index=0 + local component_payload + local infrastructure_failed=false + + KNEAD_COMPONENT_RESULTS='{}' + KNEAD_COMPONENT_RESULTS_VALID=true + + if ! component_count="$(printf '%s' "${payload}" | jq -er '.components | length')"; then + echo "failed to read component count" + KNEAD_COMPONENT_RESULTS_VALID=false + return 1 + fi + while [ "${component_index}" -lt "${component_count}" ]; do + if ! component="$(printf '%s' "${payload}" | jq -er --argjson index "${component_index}" '.components[$index].name')"; then + echo "failed to read component name at index: ${component_index}" + infrastructure_failed=true + KNEAD_COMPONENT_RESULTS_VALID=false + component_index=$((component_index + 1)) + continue + fi + if ! component_payload="$(printf '%s' "${payload}" | jq -er --argjson index "${component_index}" '.components[$index].nodeConfig')"; then + echo "failed to read component payload: ${component}" + failed_components="${failed_components} ${component}" + infrastructure_failed=true + KNEAD_COMPONENT_RESULTS_VALID=false + if ! knead_set_component_result "${component}" "Failed"; then + infrastructure_failed=true + fi + component_index=$((component_index + 1)) + continue + fi + echo "applying component: ${component}" + + # Select the handler for this component; unsupported components are ignored. + case "${component}" in + securityPatch) + component_comparator=securityPatchIsCurrent + component_handler=updateSecurityPatch + ;; + localDNS) + component_comparator=localDNSIsCurrent + component_handler=updateLocalDNS + ;; + *) + echo "unsupported component: ${component}" + component_index=$((component_index + 1)) + continue + ;; + esac + + # Skip previously applied work; otherwise apply the component and checkpoint success. + if knead_component_is_current "${component}" "${component_payload}" "${node_json}" "${component_comparator}"; then + echo "component is already current: ${component}" + if ! knead_set_component_result "${component}" "Succeeded"; then + infrastructure_failed=true + fi + elif ! "${component_handler}" "${component_payload}" "${node_json}"; then + failed_components="${failed_components} ${component}" + echo "component failed: ${component}" + if ! knead_set_component_result "${component}" "Failed"; then + infrastructure_failed=true + fi + elif ! knead_write_component_state "${component}" "${component_payload}"; then + failed_components="${failed_components} ${component}" + echo "failed to persist component state: ${component}" + if ! knead_set_component_result "${component}" "Failed"; then + infrastructure_failed=true + fi + elif ! knead_set_component_result "${component}" "Succeeded"; then + infrastructure_failed=true + fi + component_index=$((component_index + 1)) + done + + if [ -n "${failed_components}" ]; then + echo "failed components:${failed_components}" + fi + if [ "${infrastructure_failed}" = true ]; then + echo "component dispatch encountered internal failures" + fi + [ -z "${failed_components}" ] && [ "${infrastructure_failed}" = false ] +} + +# Records the processed hash and per-component results in the node status annotation. +localDNSIsCurrent() { + local desired_payload="$1" + local current_payload="$2" + + [ "${desired_payload}" = "${current_payload}" ] +} + +updateLocalDNS() { + local component_payload="${1:-}" + local outcome + + if [ ! -x /opt/azure/containers/aks-node-controller ]; then + echo "aks-node-controller binary is required for localDNS live patching" + return 1 + fi + + if ! outcome="$(printf '%s' "${component_payload}" | /opt/azure/containers/aks-node-controller apply-localdns-config \ + --config-file - \ + --output /opt/azure/containers/localdns/livepatched.localdns.corefile)"; then + echo "localDNS config apply failed" + return 1 fi + printf '%s\n' "${outcome}" - code_name=$(lsb_release -cs) - generate_sources_list "${repo_endpoint}" "${golden_timestamp}" "${code_name}" + case "$(printf '%s\n' "${outcome}" | tail -n 1)" in + applied) + if ! systemctl restart localdns.service; then + echo "failed to restart localdns.service" + return 1 + fi + echo "localDNS update completed successfully" + ;; + alreadyCurrent) + echo "localDNS is already current" + ;; + notFound) + # In the runtime apply path the payload is supplied inline, so notFound is not expected; + # treat it as a failure so knead does not checkpoint an unapplied config as succeeded. + echo "localDNS LPS config is not available" + return 1 + ;; + noCorefileData) + echo "localDNS LPS config has no node-applicable payload" + return 1 + ;; + *) + echo "unexpected localDNS apply outcome: ${outcome}" + return 1 + ;; + esac +} - export APT_CONFIG="${SECURITY_PATCH_CONFIG_DIR}/apt.conf" +knead_write_status() { + local node_name="$1" + local goal="$2" + local status - local apt_opts="-o Acquire::http::Timeout=300 -o Acquire::https::Timeout=300 -o Acquire::Retries=3" - if ! apt_get_update_with_opts "${apt_opts}"; then - echo "apt_get_update_with_opts failed" - exit 1 + if [ "${KNEAD_COMPONENT_RESULTS_VALID}" != true ]; then + echo "refusing to write incomplete live-patching status" + return 1 fi - if ! unattended_upgrade; then - echo "unattended_upgrade failed" - exit 1 + if ! status="$(printf '%s' "${KNEAD_COMPONENT_RESULTS}" | jq -c --arg currentHash "${goal}" '{currentHash: $currentHash, components: .}')" || [ -z "${status}" ]; then + echo "failed to render live-patching status annotation" + return 1 fi - # update current timestamp - $KUBECTL annotate --overwrite node ${node_name} kubernetes.azure.com/live-patching-current-timestamp=${golden_timestamp} + # shellcheck disable=SC2086 + $KUBECTL annotate --overwrite node "${node_name}" "${KNEAD_COMPONENT_STATUS_ANNOTATION}=${status}" +} + +knead_main() { + local node_name + local node_json + local goal + local status + local payload + local result=0 + + if ! knead_wait_for_kubeconfig; then + return 1 + fi + if ! node_json="$(knead_read_node)"; then + echo "failed to read node" + return 1 + fi + if ! node_name="$(printf '%s' "${node_json}" | jq -er '.metadata.name')"; then + echo "failed to read node name" + return 1 + fi + if ! goal="$(knead_get_node_annotation "${node_json}" "${KNEAD_COMPONENT_GOAL_ANNOTATION}")"; then + echo "failed to read live-patching goal annotation" + return 1 + fi + if [ -z "${goal}" ]; then + echo "live-patching goal is not set, skip knead-component" + return 0 + fi + echo "live-patching goal is: ${goal}" - echo snapshot update completed successfully + if ! status="$(knead_get_node_annotation "${node_json}" "${KNEAD_COMPONENT_STATUS_ANNOTATION}")"; then + echo "failed to read live-patching status annotation" + return 1 + fi + # Skip reconciliation only when this goal was processed and every component succeeded. + if [ -n "${status}" ] && printf '%s' "${status}" | jq -e --arg goal "${goal}" \ + '.currentHash == $goal and (.components | type == "object") and (.components | all(.code == "Succeeded"))' \ + > /dev/null 2>&1; then + echo "live-patching goal is already converged, nothing to apply" + return 0 + fi + + # Stop before dispatch when generic inputs are missing or unsafe. Once dispatch + # starts, knead_apply_components owns the continue-after-component-failure + # behavior so one broken handler does not prevent other handlers from running. + if ! payload="$(knead_read_configmap "${goal}")"; then + result=1 + else + if ! knead_apply_components "${payload}" "${node_json}"; then + result=1 + fi + + if ! knead_write_status "${node_name}" "${goal}"; then + echo "failed to update live-patching status annotation" + result=1 + fi + fi + + if [ "${result}" -eq 0 ]; then + echo "knead-component completed successfully" + fi + return "${result}" } ${__SOURCED__:+return} # --------------------------------------- Main Execution starts here -------------------------------------------------- -# source apt_get_update -source /opt/azure/containers/provision_source_distro.sh +# shellcheck disable=SC1091 +source /opt/azure/containers/security-update.sh -main "$@" +knead_main "$@" diff --git a/parts/linux/cloud-init/nodecustomdata.yml b/parts/linux/cloud-init/nodecustomdata.yml index 096f851953e..2d714536580 100644 --- a/parts/linux/cloud-init/nodecustomdata.yml +++ b/parts/linux/cloud-init/nodecustomdata.yml @@ -154,6 +154,28 @@ write_files: content: !!binary | {{GetVariableProperty "cloudInitData" "initAKSCloud"}} +# Deliver the generic reconciler and its security handler together so replacing +# the established updater path never leaves it without the dispatched handler. +{{if IsACL }} +{{- else if IsAzlOSGuard}} +{{- else if IsMariner}} +{{- else if IsFlatcar }} +{{- else }} +- path: /opt/azure/containers/ubuntu-snapshot-update.sh + permissions: "0544" + encoding: gzip + owner: root + content: !!binary | + {{GetVariableProperty "cloudInitData" "snapshotUpdateScript"}} + +- path: /opt/azure/containers/security-update.sh + permissions: "0544" + encoding: gzip + owner: root + content: !!binary | + {{GetVariableProperty "cloudInitData" "securityUpdateScript"}} +{{end}} + - path: /etc/systemd/system/reconcile-private-hosts.service permissions: "0644" encoding: gzip diff --git a/parts/windows/kuberneteswindowssetup.ps1 b/parts/windows/kuberneteswindowssetup.ps1.template similarity index 55% rename from parts/windows/kuberneteswindowssetup.ps1 rename to parts/windows/kuberneteswindowssetup.ps1.template index 014c77e61ae..fcedc292332 100644 --- a/parts/windows/kuberneteswindowssetup.ps1 +++ b/parts/windows/kuberneteswindowssetup.ps1.template @@ -36,19 +36,20 @@ param( $CSEResultFilePath ) + # In an ideal world, all these values would be passed to this script in parameters. However, we don't live in an ideal world. # https://learn.microsoft.com/en-gb/troubleshoot/windows-client/shell-experience/command-line-string-limitation -$MasterIP = "{{ GetKubernetesEndpoint }}" -$KubeDnsServiceIp="{{ GetParameter "kubeDNSServiceIP" }}" -$MasterFQDNPrefix="{{ GetParameter "masterEndpointDNSNamePrefix" }}" -$Location="{{ GetVariable "location" }}" +$MasterIP="{{GetKubernetesEndpoint}}" +$KubeDnsServiceIp="{{GetParameter "kubeDNSServiceIP"}}" +$MasterFQDNPrefix="{{GetParameter "masterEndpointDNSNamePrefix"}}" +$Location="{{GetVariable "location"}}" {{if UserAssignedIDEnabled}} -$UserAssignedClientID="{{ GetVariable "userAssignedIdentityID" }}" -{{ end }} -$TargetEnvironment="{{ GetTargetEnvironment }}" -$ArmResourceEndpoint="{{ GetArmResourceEndpoint }}" -$AADClientId="{{ GetParameter "servicePrincipalClientId" }}" +$UserAssignedClientID="{{GetVariable "userAssignedIdentityID"}}" +{{end}} +$TargetEnvironment="{{GetTargetEnvironment}}" +$ArmResourceEndpoint="{{GetArmResourceEndpoint}}" +$AADClientId="{{GetParameter "servicePrincipalClientId"}}" $NetworkAPIVersion="2018-08-01" # Do not parse the start time from $LogFile to simplify the logic @@ -60,293 +61,301 @@ $global:ErrorMessage="" # passed as powershell parameters ## SSH public keys to add to authorized_keys -$global:SSHKeys = @( {{ GetSshPublicKeysPowerShell }} ) +$global:SSHKeys=@( {{GetSshPublicKeysPowerShell}} ) ## Certificates generated by aks-engine -$global:CACertificate = "{{GetParameter "caCertificate"}}" -$global:AgentCertificate = "{{GetParameter "clientCertificate"}}" +$global:CACertificate="{{GetParameter "caCertificate"}}" +$global:AgentCertificate="{{GetParameter "clientCertificate"}}" ## Download sources provided by aks-engine -$global:KubeBinariesPackageSASURL = "{{GetParameter "kubeBinariesSASURL"}}" -$global:WindowsKubeBinariesURL = "{{GetParameter "windowsKubeBinariesURL"}}" -$global:KubeBinariesVersion = "{{GetParameter "kubeBinariesVersion"}}" -$global:ContainerdUrl = "{{GetParameter "windowsContainerdURL"}}" -$global:ContainerdSdnPluginUrl = "{{GetParameter "windowsSdnPluginURL"}}" +$global:KubeBinariesPackageSASURL="{{GetParameter "kubeBinariesSASURL"}}" +$global:WindowsKubeBinariesURL="{{GetParameter "windowsKubeBinariesURL"}}" +$global:KubeBinariesVersion="{{GetParameter "kubeBinariesVersion"}}" +$global:ContainerdUrl="{{GetParameter "windowsContainerdURL"}}" +$global:ContainerdSdnPluginUrl="{{GetParameter "windowsSdnPluginURL"}}" ## Docker Version -$global:DockerVersion = "{{GetParameter "windowsDockerVersion"}}" +$global:DockerVersion="{{GetParameter "windowsDockerVersion"}}" ## ContainerD Usage -$global:DefaultContainerdWindowsSandboxIsolation = "{{GetParameter "defaultContainerdWindowsSandboxIsolation"}}" -$global:ContainerdWindowsRuntimeHandlers = "{{GetParameter "containerdWindowsRuntimeHandlers"}}" +$global:DefaultContainerdWindowsSandboxIsolation="{{GetParameter "defaultContainerdWindowsSandboxIsolation"}}" +$global:ContainerdWindowsRuntimeHandlers="{{GetParameter "containerdWindowsRuntimeHandlers"}}" ## VM configuration passed by Azure -$global:WindowsTelemetryGUID = "{{GetParameter "windowsTelemetryGUID"}}" +$global:WindowsTelemetryGUID="{{GetParameter "windowsTelemetryGUID"}}" {{if eq GetIdentitySystem "adfs"}} -$global:TenantId = "adfs" +$global:TenantId="adfs" {{else}} -$global:TenantId = "{{GetVariable "tenantID"}}" +$global:TenantId="{{GetVariable "tenantID"}}" {{end}} -$global:SubscriptionId = "{{GetVariable "subscriptionId"}}" -$global:ResourceGroup = "{{GetVariable "resourceGroup"}}" -$global:VmType = "{{GetVariable "vmType"}}" -$global:SubnetName = "{{GetVariable "subnetName"}}" +$global:SubscriptionId="{{GetVariable "subscriptionId"}}" +$global:ResourceGroup="{{GetVariable "resourceGroup"}}" +$global:VmType="{{GetVariable "vmType"}}" +$global:SubnetName="{{GetVariable "subnetName"}}" # NOTE: MasterSubnet is still referenced by `kubeletstart.ps1` and `windowsnodereset.ps1` # for case of Kubenet -$global:MasterSubnet = "" -$global:SecurityGroupName = "{{GetVariable "nsgName"}}" -$global:VNetName = "{{GetVariable "virtualNetworkName"}}" -$global:RouteTableName = "{{GetVariable "routeTableName"}}" -$global:PrimaryAvailabilitySetName = "{{GetVariable "primaryAvailabilitySetName"}}" -$global:PrimaryScaleSetName = "{{GetVariable "primaryScaleSetName"}}" - -$global:KubeClusterCIDR = "{{GetParameter "kubeClusterCidr"}}" -$global:KubeServiceCIDR = "{{GetParameter "kubeServiceCidr"}}" -$global:VNetCIDR = "{{GetParameter "vnetCidr"}}" +$global:MasterSubnet="" +$global:SecurityGroupName="{{GetVariable "nsgName"}}" +$global:VNetName="{{GetVariable "virtualNetworkName"}}" +$global:RouteTableName="{{GetVariable "routeTableName"}}" +$global:PrimaryAvailabilitySetName="{{GetVariable "primaryAvailabilitySetName"}}" +$global:PrimaryScaleSetName="{{GetVariable "primaryScaleSetName"}}" + +$global:KubeClusterCIDR="{{GetParameter "kubeClusterCidr"}}" +$global:KubeServiceCIDR="{{GetParameter "kubeServiceCidr"}}" +$global:VNetCIDR="{{GetParameter "vnetCidr"}}" {{if IsKubernetesVersionGe "1.16.0"}} -$global:KubeletNodeLabels = "{{GetAgentKubernetesLabels . }}" +$global:KubeletNodeLabels="{{GetAgentKubernetesLabels .}}" {{else}} -$global:KubeletNodeLabels = "{{GetAgentKubernetesLabelsDeprecated . }}" +$global:KubeletNodeLabels="{{GetAgentKubernetesLabelsDeprecated .}}" {{end}} -$global:KubeletConfigArgs = @( {{GetKubeletConfigKeyValsPsh}} ) -$global:KubeproxyConfigArgs = @( {{GetKubeproxyConfigKeyValsPsh}} ) +$global:KubeletConfigArgs=@( {{GetKubeletConfigKeyValsPsh}} ) +$global:KubeproxyConfigArgs=@( {{GetKubeproxyConfigKeyValsPsh}} ) -$global:KubeproxyFeatureGates = @( {{GetKubeProxyFeatureGatesPsh}} ) +$global:KubeproxyFeatureGates=@( {{GetKubeProxyFeatureGatesPsh}} ) -$global:UseManagedIdentityExtension = "{{GetVariable "useManagedIdentityExtension"}}" -$global:UseInstanceMetadata = "{{GetVariable "useInstanceMetadata"}}" +$global:UseManagedIdentityExtension="{{GetVariable "useManagedIdentityExtension"}}" +$global:UseInstanceMetadata="{{GetVariable "useInstanceMetadata"}}" -$global:LoadBalancerSku = "{{GetVariable "loadBalancerSku"}}" -$global:ExcludeMasterFromStandardLB = "{{GetVariable "excludeMasterFromStandardLB"}}" +$global:LoadBalancerSku="{{GetVariable "loadBalancerSku"}}" +$global:ExcludeMasterFromStandardLB="{{GetVariable "excludeMasterFromStandardLB"}}" -$global:PrivateEgressProxyAddress = "{{GetPrivateEgressProxyAddress}}" +$global:PrivateEgressProxyAddress="{{GetPrivateEgressProxyAddress}}" # Windows defaults, not changed by aks-engine -$global:CacheDir = "c:\akse-cache" -$global:KubeDir = "c:\k" -$global:HNSModule = [Io.path]::Combine("$global:KubeDir", "hns.v2.psm1") +$global:CacheDir="c:\akse-cache" +$global:KubeDir="c:\k" +$global:HNSModule=[Io.path]::Combine("$global:KubeDir", "hns.v2.psm1") -$global:KubeDnsSearchPath = "svc.cluster.local" +$global:KubeDnsSearchPath="svc.cluster.local" -$global:CNIPath = [Io.path]::Combine("$global:KubeDir", "cni") -$global:NetworkMode = "L2Bridge" -$global:CNIConfig = [Io.path]::Combine($global:CNIPath, "config", "`$global:NetworkMode.conf") -$global:CNIConfigPath = [Io.path]::Combine("$global:CNIPath", "config") +$global:CNIPath=[Io.path]::Combine("$global:KubeDir", "cni") +$global:NetworkMode="L2Bridge" +$global:CNIConfig=[Io.path]::Combine($global:CNIPath, "config", "`$global:NetworkMode.conf") +$global:CNIConfigPath=[Io.path]::Combine("$global:CNIPath", "config") -$global:AzureCNIDir = [Io.path]::Combine("$global:KubeDir", "azurecni") -$global:AzureCNIBinDir = [Io.path]::Combine("$global:AzureCNIDir", "bin") -$global:AzureCNIConfDir = [Io.path]::Combine("$global:AzureCNIDir", "netconf") +$global:AzureCNIDir=[Io.path]::Combine("$global:KubeDir", "azurecni") +$global:AzureCNIBinDir=[Io.path]::Combine("$global:AzureCNIDir", "bin") +$global:AzureCNIConfDir=[Io.path]::Combine("$global:AzureCNIDir", "netconf") # Azure cni configuration -# $global:NetworkPolicy = "{{GetParameter "networkPolicy"}}" # BUG: unused -$global:NetworkPlugin = "{{GetParameter "networkPlugin"}}" -$global:VNetCNIPluginsURL = "{{GetParameter "vnetCniWindowsPluginsURL"}}" -$global:IsDualStackEnabled = {{if IsIPv6DualStackFeatureEnabled}}$true{{else}}$false{{end}} -$global:IsAzureCNIOverlayEnabled = {{if IsAzureCNIOverlayFeatureEnabled}}$true{{else}}$false{{end}} -$global:CiliumDataplaneEnabled = {{if CiliumDataplaneEnabled}}$true{{else}}$false{{end}} -$global:IsIMDSRestrictionEnabled = {{if EnableIMDSRestriction}}$true{{else}}$false{{end}} +# $global:NetworkPolicy="{{GetParameter "networkPolicy"}}" # BUG: unused +$global:NetworkPlugin="{{GetParameter "networkPlugin"}}" +$global:VNetCNIPluginsURL="{{GetParameter "vnetCniWindowsPluginsURL"}}" +$global:IsDualStackEnabled={{if IsIPv6DualStackFeatureEnabled}}$true {{else}}$false {{end}} +$global:IsAzureCNIOverlayEnabled={{if IsAzureCNIOverlayFeatureEnabled}}$true {{else}}$false {{end}} +$global:CiliumDataplaneEnabled={{if CiliumDataplaneEnabled}}$true {{else}}$false {{end}} +$global:IsIMDSRestrictionEnabled={{if EnableIMDSRestriction}}$true {{else}}$false {{end}} # Kubelet credential provider -$global:CredentialProviderURL = "{{GetParameter "windowsCredentialProviderURL"}}" +$global:CredentialProviderURL="{{GetParameter "windowsCredentialProviderURL"}}" # CSI Proxy settings -$global:EnableCsiProxy = [System.Convert]::ToBoolean("{{GetVariable "windowsEnableCSIProxy" }}"); -$global:CsiProxyUrl = "{{GetVariable "windowsCSIProxyURL" }}"; +$global:EnableCsiProxy=[System.Convert]::ToBoolean("{{GetVariable "windowsEnableCSIProxy"}}"); +$global:CsiProxyUrl="{{GetVariable "windowsCSIProxyURL"}}"; # Hosts Config Agent settings -$global:EnableHostsConfigAgent = [System.Convert]::ToBoolean("{{ EnableHostsConfigAgent }}"); +$global:EnableHostsConfigAgent=[System.Convert]::ToBoolean("{{EnableHostsConfigAgent}}"); # These scripts are used by cse -$global:CSEScriptsPackageUrl = "{{GetVariable "windowsCSEScriptsPackageURL" }}"; +$global:CSEScriptsPackageUrl="{{GetVariable "windowsCSEScriptsPackageURL"}}"; # The windows nvidia gpu driver related url is used by windows cse -$global:GpuDriverURL = "{{GetVariable "windowsGpuDriverURL" }}"; +$global:GpuDriverURL="{{GetVariable "windowsGpuDriverURL"}}"; # PauseImage -$global:WindowsPauseImageURL = "{{GetVariable "windowsPauseImageURL" }}"; -$global:AlwaysPullWindowsPauseImage = [System.Convert]::ToBoolean("{{GetVariable "alwaysPullWindowsPauseImage" }}"); +$global:WindowsPauseImageURL="{{GetVariable "windowsPauseImageURL"}}"; +$global:AlwaysPullWindowsPauseImage=[System.Convert]::ToBoolean("{{GetVariable "alwaysPullWindowsPauseImage"}}"); # Calico -$global:WindowsCalicoPackageURL = "{{GetVariable "windowsCalicoPackageURL" }}"; +$global:WindowsCalicoPackageURL="{{GetVariable "windowsCalicoPackageURL"}}"; ## GPU install -$global:ConfigGPUDriverIfNeeded = [System.Convert]::ToBoolean("{{GetVariable "configGPUDriverIfNeeded" }}"); +$global:ConfigGPUDriverIfNeeded=[System.Convert]::ToBoolean("{{GetVariable "configGPUDriverIfNeeded"}}"); # GMSA -$global:WindowsGmsaPackageUrl = "{{GetVariable "windowsGmsaPackageUrl" }}"; +$global:WindowsGmsaPackageUrl="{{GetVariable "windowsGmsaPackageUrl"}}"; # TLS Bootstrap Token -$global:TLSBootstrapToken = "{{GetTLSBootstrapTokenForKubeConfig}}" +$global:TLSBootstrapToken="{{GetTLSBootstrapTokenForKubeConfig}}" # Secure TLS Bootstrap settings -$global:EnableSecureTLSBootstrapping = [System.Convert]::ToBoolean("{{EnableSecureTLSBootstrapping}}"); -$global:SecureTLSBootstrappingAADResource = "{{GetSecureTLSBootstrappingAADResource}}"; -$global:SecureTLSBootstrappingUserAssignedIdentityID = "{{GetSecureTLSBootstrappingUserAssignedIdentityID}}"; -$global:CustomSecureTLSBootstrappingClientDownloadURL = "{{GetCustomSecureTLSBootstrappingClientDownloadURL}}"; -$global:SecureTLSBootstrappingValidateKubeconfigTimeout = "{{GetSecureTLSBootstrappingValidateKubeconfigTimeout}}"; -$global:SecureTLSBootstrappingGetAccessTokenTimeout = "{{GetSecureTLSBootstrappingGetAccessTokenTimeout}}"; -$global:SecureTLSBootstrappingGetInstanceDataTimeout = "{{GetSecureTLSBootstrappingGetInstanceDataTimeout}}"; -$global:SecureTLSBootstrappingGetNonceTimeout = "{{GetSecureTLSBootstrappingGetNonceTimeout}}"; -$global:SecureTLSBootstrappingGetAttestedDataTimeout = "{{GetSecureTLSBootstrappingGetAttestedDataTimeout}}"; -$global:SecureTLSBootstrappingGetCredentialTimeout = "{{GetSecureTLSBootstrappingGetCredentialTimeout}}"; +$global:EnableSecureTLSBootstrapping=[System.Convert]::ToBoolean("{{EnableSecureTLSBootstrapping}}"); +$global:SecureTLSBootstrappingAADResource="{{GetSecureTLSBootstrappingAADResource}}"; +$global:SecureTLSBootstrappingUserAssignedIdentityID="{{GetSecureTLSBootstrappingUserAssignedIdentityID}}"; +$global:CustomSecureTLSBootstrappingClientDownloadURL="{{GetCustomSecureTLSBootstrappingClientDownloadURL}}"; +$global:SecureTLSBootstrappingValidateKubeconfigTimeout="{{GetSecureTLSBootstrappingValidateKubeconfigTimeout}}"; +$global:SecureTLSBootstrappingGetAccessTokenTimeout="{{GetSecureTLSBootstrappingGetAccessTokenTimeout}}"; +$global:SecureTLSBootstrappingGetInstanceDataTimeout="{{GetSecureTLSBootstrappingGetInstanceDataTimeout}}"; +$global:SecureTLSBootstrappingGetNonceTimeout="{{GetSecureTLSBootstrappingGetNonceTimeout}}"; +$global:SecureTLSBootstrappingGetAttestedDataTimeout="{{GetSecureTLSBootstrappingGetAttestedDataTimeout}}"; +$global:SecureTLSBootstrappingGetCredentialTimeout="{{GetSecureTLSBootstrappingGetCredentialTimeout}}"; # uniquely identifies AKS's Entra ID application, see: https://learn.microsoft.com/en-us/azure/aks/kubelogin-authentication#how-to-use-kubelogin-with-aks # this is used by aks-secure-tls-bootstrap-client.exe when requesting AAD tokens # TODO(cameissner): remove once 2025-10B image is released -$global:AKSAADServerAppID = "6dae42f8-4368-4678-94ff-3960e28e3630" +$global:AKSAADServerAppID="6dae42f8-4368-4678-94ff-3960e28e3630" # Disable OutBoundNAT in Azure CNI configuration -$global:IsDisableWindowsOutboundNat = [System.Convert]::ToBoolean("{{GetVariable "isDisableWindowsOutboundNat" }}"); +$global:IsDisableWindowsOutboundNat=[System.Convert]::ToBoolean("{{GetVariable "isDisableWindowsOutboundNat"}}"); # Base64 representation of ZIP archive -$zippedFiles = "{{ GetKubernetesWindowsAgentFunctions }}" +$zippedFiles="{{GetKubernetesWindowsAgentFunctions}}" -$global:KubeClusterConfigPath = "c:\k\kubeclusterconfig.json" -$fipsEnabled = [System.Convert]::ToBoolean("{{ FIPSEnabled }}") +$global:KubeClusterConfigPath="c:\k\kubeclusterconfig.json" +$fipsEnabled=[System.Convert]::ToBoolean("{{FIPSEnabled}}") # HNS remediator -$global:HNSRemediatorIntervalInMinutes = [System.Convert]::ToUInt32("{{GetHnsRemediatorIntervalInMinutes}}"); +$global:HNSRemediatorIntervalInMinutes=[System.Convert]::ToUInt32("{{GetHnsRemediatorIntervalInMinutes}}"); # Log generator -$global:LogGeneratorIntervalInMinutes = [System.Convert]::ToUInt32("{{GetLogGeneratorIntervalInMinutes}}"); +$global:LogGeneratorIntervalInMinutes=[System.Convert]::ToUInt32("{{GetLogGeneratorIntervalInMinutes}}"); -$global:EnableIncreaseDynamicPortRange = $false +$global:EnableIncreaseDynamicPortRange=$false -$global:RebootNeeded = $false +$global:RebootNeeded=$false -$global:IsSkipCleanupNetwork = [System.Convert]::ToBoolean("{{GetVariable "isSkipCleanupNetwork" }}"); -$PreProvisionOnly = [System.Convert]::ToBoolean("{{GetPreProvisionOnly}}"); +$global:IsSkipCleanupNetwork=[System.Convert]::ToBoolean("{{GetVariable "isSkipCleanupNetwork"}}"); +$PreProvisionOnly=[System.Convert]::ToBoolean("{{GetPreProvisionOnly}}"); -$global:EnableKubeletServingCertificateRotation = [System.Convert]::ToBoolean("{{EnableKubeletServingCertificateRotation}}") +$global:EnableKubeletServingCertificateRotation=[System.Convert]::ToBoolean("{{EnableKubeletServingCertificateRotation}}") # Windows Cilium Networking (WCN) Platform configuration -$global:EnableWindowsCiliumNetworking = [System.Convert]::ToBoolean("{{GetVariable "nextGenNetworkingEnabled" }}"); -$global:WindowsCiliumNetworkingConfiguration = "{{GetVariable "nextGenNetworkingConfig" }}"; -$global:WindowsCiliumNetworkingPath = Join-Path -Path $global:cacheDir -ChildPath 'wcn' -$global:WindowsCiliumInstallPath = Join-Path -Path $global:WindowsCiliumNetworkingPath -ChildPath 'install' +$global:EnableWindowsCiliumNetworking=[System.Convert]::ToBoolean("{{GetVariable "nextGenNetworkingEnabled"}}"); +$global:WindowsCiliumNetworkingConfiguration="{{GetVariable "nextGenNetworkingConfig"}}"; +$global:WindowsCiliumNetworkingPath=Join-Path -Path $global:cacheDir -ChildPath 'wcn' +$global:WindowsCiliumInstallPath=Join-Path -Path $global:WindowsCiliumNetworkingPath -ChildPath 'install' # Network isolated cluster $global:BootstrapProfileContainerRegistryServer="{{GetBootstrapProfileContainerRegistryServer}}" $global:MCRRepositoryBase="{{GetMCRRepositoryBase}}" -$global:NetworkIsolatedClusterTestMode = [System.Convert]::ToBoolean("{{GetNetworkIsolatedClusterTestMode}}"); # for ab e2e only for local ab test with remote cse package +$global:NetworkIsolatedClusterTestMode=[System.Convert]::ToBoolean("{{GetNetworkIsolatedClusterTestMode}}"); # for ab e2e only for local ab test with remote cse package $global:OrasCacheDir="c:\aks-tools\oras\" # refer to components.json $global:OrasPath="c:\aks-tools\oras\oras.exe" $global:OrasOutput="c:\aks-tools\oras\oras_verbose.out" $global:OrasRegistryConfigFile="c:\aks-tools\oras\config.yaml" # oras registry auth config file, not used, but have to define to avoid error "Error: failed to get user home directory: $HOME is not defined" +$global:OperationId=New-Guid # Extract cse helper script from ZIP -[io.file]::WriteAllBytes("scripts.zip", [System.Convert]::FromBase64String($zippedFiles)) -try { - Expand-Archive scripts.zip -DestinationPath "C:\\AzureData\\" -Force -ErrorAction Stop -} catch { - $global:ErrorMessage = ("Failed to extract inline scripts.zip: $($_.Exception.Message)" -replace '\|', '%7C' ) - $global:ExitCode = 73 - exit 73 -} +function Get-HelperScripts { + [io.file]::WriteAllBytes("scripts.zip", [System.Convert]::FromBase64String($zippedFiles)) + try { + Expand-Archive scripts.zip -DestinationPath "C:\\AzureData\\" -Force -ErrorAction Stop + # Dot-source windowscsehelper.ps1 with functions that are called in this script + . c:\AzureData\windows\windowscsehelper.ps1 + # util functions only can be used after this line, for example, Write-Log + } + catch { + $global:ErrorMessage=("Failed to extract inline scripts.zip: $($_.Exception.Message)" -replace '\|', '%7C' ) + $global:ExitCode=73 + exit 73 + } -# Dot-source windowscsehelper.ps1 with functions that are called in this script -. c:\AzureData\windows\windowscsehelper.ps1 -# util functions only can be used after this line, for example, Write-Log - -$global:OperationId = New-Guid - -# Detect whether the operator has supplied a fully-qualified CSE scripts zip URL -# (as opposed to a base URL ending in "/"). A fully-qualified URL is a signal to -# *always* fetch and overwrite the cached scripts on the VHD, which is required -# for testing branch builds against baked VHDs that already contain a prior -# version of the CSE scripts. Production RP only ever sets a base URL ending in -# "/", so this branch is a no-op for production traffic. -$global:IsExplicitCSEScriptsPackageUrl = $false -if (-not [string]::IsNullOrWhiteSpace($global:CSEScriptsPackageUrl) -and -not $global:CSEScriptsPackageUrl.EndsWith("/")) { - $global:IsExplicitCSEScriptsPackageUrl = $true -} + # Detect whether the operator has supplied a fully-qualified CSE scripts zip URL + # (as opposed to a base URL ending in "/"). A fully-qualified URL is a signal to + # *always* fetch and overwrite the cached scripts on the VHD, which is required + # for testing branch builds against baked VHDs that already contain a prior + # version of the CSE scripts. Production RP only ever sets a base URL ending in + # "/", so this branch is a no-op for production traffic. + $global:IsExplicitCSEScriptsPackageUrl=$false + if (-not [string]::IsNullOrWhiteSpace($global:CSEScriptsPackageUrl) -and -not $global:CSEScriptsPackageUrl.EndsWith("/")) { + $global:IsExplicitCSEScriptsPackageUrl=$true + } -if ((-not (Test-Path "C:\AzureData\windows\azurecnifunc.ps1")) -or $global:IsExplicitCSEScriptsPackageUrl) { - # CSEScriptsPackage is cached on VHD. Previously the cse package version was managed in components.json, whereas RP set the package URL which is a storage account. - # From 2025-06 The CSE packages is released on the VHD. RP can use fully qualified URL to download CSE scripts package when required out of VHD release cycle. - # In the transition period, it is important that when deal with older VHD versions, the agentbaker runtime provision script needs to be compatible with the latest known storage account package, 0.0.52. - - $WindowsCSEScriptsPackage = "aks-windows-cse-scripts-current.zip" - $scriptsZip = $null - $shouldCleanup = $false - - # Step 1: Try to find cached scripts on VHD (skipped when an explicit - # CSEScriptsPackageUrl was supplied, so the operator-provided zip is the - # source of truth and always overwrites the cached copy). - if (-not $global:IsExplicitCSEScriptsPackageUrl -and $global:CacheDir -and (Test-Path $global:CacheDir)) { - $searchCachedScripts = [IO.Directory]::GetFiles($global:CacheDir, $WindowsCSEScriptsPackage, [IO.SearchOption]::AllDirectories) - Write-Log "the directory $global:CacheDir contains the following files:" - Get-ChildItem -Path $global:CacheDir | ForEach-Object { Write-Log " $_" } - if ($searchCachedScripts.Count -gt 0) { - $scriptsZip = $searchCachedScripts[0] - Write-Log "Found cached CSE scripts at $scriptsZip" + if ((-not (Test-Path "C:\AzureData\windows\azurecnifunc.ps1")) -or $global:IsExplicitCSEScriptsPackageUrl) { + # CSEScriptsPackage is cached on VHD. Previously the cse package version was managed in components.json, whereas RP set the package URL which is a storage account. + # From 2025-06 The CSE packages is released on the VHD. RP can use fully qualified URL to download CSE scripts package when required out of VHD release cycle. + # In the transition period, it is important that when deal with older VHD versions, the agentbaker runtime provision script needs to be compatible with the latest known storage account package, 0.0.52. + + $WindowsCSEScriptsPackage="aks-windows-cse-scripts-current.zip" + $scriptsZip=$null + $shouldCleanup=$false + + # Step 1: Try to find cached scripts on VHD (skipped when an explicit + # CSEScriptsPackageUrl was supplied, so the operator-provided zip is the + # source of truth and always overwrites the cached copy). + if (-not $global:IsExplicitCSEScriptsPackageUrl -and $global:CacheDir -and (Test-Path $global:CacheDir)) { + $searchCachedScripts=[IO.Directory]::GetFiles($global:CacheDir, $WindowsCSEScriptsPackage, [IO.SearchOption]::AllDirectories) + Write-Log "the directory $global:CacheDir contains the following files:" + Get-ChildItem -Path $global:CacheDir | ForEach-Object { Write-Log " $_" } + if ($searchCachedScripts.Count -gt 0) { + $scriptsZip=$searchCachedScripts[0] + Write-Log "Found cached CSE scripts at $scriptsZip" + } } - } - # Step 2: For non-network-isolated clusters, download scripts if needed (overrides cached version when appropriate) - $isNetworkIsolated = -not [string]::IsNullOrWhiteSpace($global:BootstrapProfileContainerRegistryServer) -and -not $global:NetworkIsolatedClusterTestMode - if (-not $isNetworkIsolated) { - Write-Log "Requested CSEScriptsPackageUrl is $global:CSEScriptsPackageUrl" - if ($global:CSEScriptsPackageUrl.EndsWith("/")) { + # Step 2: For non-network-isolated clusters, download scripts if needed (overrides cached version when appropriate) + $isNetworkIsolated=-not [string]::IsNullOrWhiteSpace($global:BootstrapProfileContainerRegistryServer) -and -not $global:NetworkIsolatedClusterTestMode + if (-not $isNetworkIsolated) { + Write-Log "Requested CSEScriptsPackageUrl is $global:CSEScriptsPackageUrl" + if ($global:CSEScriptsPackageUrl.EndsWith("/")) { + if (-not $scriptsZip) { + Write-Log "Could not find windows cse package on VHD. Use remote version instead." + $WindowsCSEScriptsPackage="aks-windows-cse-scripts-v0.0.52.zip" + } + Write-Log "WindowsCSEScriptsPackage is $WindowsCSEScriptsPackage" + $global:CSEScriptsPackageUrl=$global:CSEScriptsPackageUrl + $WindowsCSEScriptsPackage + } + Write-Log "CSEScriptsPackageUrl used for provision is $global:CSEScriptsPackageUrl" + + # Download CSE function scripts + $downloadedFile='c:\csescripts.zip' + Logs-To-Event -TaskName "AKS.WindowsCSE.DownloadAndExpandCSEScriptPackageUrl" -TaskMessage "Start to get CSE scripts. CSEScriptsPackageUrl: $global:CSEScriptsPackageUrl" + DownloadFileOverHttp -Url $global:CSEScriptsPackageUrl -DestinationPath $downloadedFile -ExitCode $global:WINDOWS_CSE_ERROR_DOWNLOAD_CSE_PACKAGE + $scriptsZip=$downloadedFile + $shouldCleanup=$true + } + else { + Write-Log "Network isolated cluster detected (BootstrapProfileContainerRegistryServer is set), skip CSE scripts download and use cached scripts" if (-not $scriptsZip) { - Write-Log "Could not find windows cse package on VHD. Use remote version instead." - $WindowsCSEScriptsPackage = "aks-windows-cse-scripts-v0.0.52.zip" + Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_NETWORK_ISOLATED_CLUSTER_CSE_NOT_CACHED -ErrorMessage "Cached CSE scripts package '$WindowsCSEScriptsPackage' not found under cache directory '$global:CacheDir'" } - Write-Log "WindowsCSEScriptsPackage is $WindowsCSEScriptsPackage" - $global:CSEScriptsPackageUrl = $global:CSEScriptsPackageUrl + $WindowsCSEScriptsPackage } - Write-Log "CSEScriptsPackageUrl used for provision is $global:CSEScriptsPackageUrl" - - # Download CSE function scripts - $downloadedFile = 'c:\csescripts.zip' - Logs-To-Event -TaskName "AKS.WindowsCSE.DownloadAndExpandCSEScriptPackageUrl" -TaskMessage "Start to get CSE scripts. CSEScriptsPackageUrl: $global:CSEScriptsPackageUrl" - DownloadFileOverHttp -Url $global:CSEScriptsPackageUrl -DestinationPath $downloadedFile -ExitCode $global:WINDOWS_CSE_ERROR_DOWNLOAD_CSE_PACKAGE - $scriptsZip = $downloadedFile - $shouldCleanup = $true - } else { - Write-Log "Network isolated cluster detected (BootstrapProfileContainerRegistryServer is set), skip CSE scripts download and use cached scripts" - if (-not $scriptsZip) { - Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_NETWORK_ISOLATED_CLUSTER_CSE_NOT_CACHED -ErrorMessage "Cached CSE scripts package '$WindowsCSEScriptsPackage' not found under cache directory '$global:CacheDir'" + + # Step 3: Extract scripts from the resolved zip + Write-Log "Extracting CSE scripts from $scriptsZip" + AKS-Expand-Archive -Path $scriptsZip -DestinationPath "C:\\AzureData\\windows" + if ($shouldCleanup) { + Remove-Item -Path $scriptsZip -Force } } + else { + Write-Log "CSE scripts already exist and no explicit CSEScriptsPackageUrl override, skipping download" + } - # Step 3: Extract scripts from the resolved zip - Write-Log "Extracting CSE scripts from $scriptsZip" - AKS-Expand-Archive -Path $scriptsZip -DestinationPath "C:\\AzureData\\windows" - if ($shouldCleanup) { - Remove-Item -Path $scriptsZip -Force + if (Test-Path -Path 'c:\AzureData\windows\all.ps1') { + . c:\AzureData\windows\all.ps1 + return } -} else { - Write-Log "CSE scripts already exist and no explicit CSEScriptsPackageUrl override, skipping download" -} -# Dot-source cse scripts with functions that are called in this script -. c:\AzureData\windows\azurecnifunc.ps1 -. c:\AzureData\windows\calicofunc.ps1 -. c:\AzureData\windows\configfunc.ps1 -. c:\AzureData\windows\containerdfunc.ps1 -. c:\AzureData\windows\kubeletfunc.ps1 -. c:\AzureData\windows\kubernetesfunc.ps1 -. c:\AzureData\windows\nvidiagpudriverfunc.ps1 - -if (Test-Path -Path 'c:\AzureData\windows\securetlsbootstrapfunc.ps1') { - . c:\AzureData\windows\securetlsbootstrapfunc.ps1 -} else { - Write-Log "Windows Secure TLS Bootstrap function script not found, skipping dot-source" -} + Write-Log "All CSE function script not found, falling back to individual import" -if (Test-Path -Path 'c:\AzureData\windows\windowsciliumnetworkingfunc.ps1') { - . c:\AzureData\windows\windowsciliumnetworkingfunc.ps1 -} else { - Write-Log "Windows Cilium Networking function script not found, skipping dot-source" -} + # Dot-source cse scripts with functions that are called in this script -if (Test-Path -Path 'c:\AzureData\windows\networkisolatedclusterfunc.ps1') { - . c:\AzureData\windows\networkisolatedclusterfunc.ps1 -} else { - Write-Log "Network Isolated Cluster function script not found, skipping dot-source" + # Dot-source cse scripts with functions that are called in this script + . c:\AzureData\windows\azurecnifunc.ps1 + . c:\AzureData\windows\calicofunc.ps1 + . c:\AzureData\windows\configfunc.ps1 + . c:\AzureData\windows\containerdfunc.ps1 + . c:\AzureData\windows\kubeletfunc.ps1 + . c:\AzureData\windows\kubernetesfunc.ps1 + . c:\AzureData\windows\nvidiagpudriverfunc.ps1 + + $optionalFunctionScripts = @( + 'c:\AzureData\windows\securetlsbootstrapfunc.ps1' + 'c:\AzureData\windows\windowsciliumnetworkingfunc.ps1' + 'c:\AzureData\windows\networkisolatedclusterfunc.ps1' + ) + + foreach ($script in $optionalFunctionScripts) { + if (Test-Path -Path $script) { + . $script + } else { + Write-Log "$script function script not found, skipping dot-source" + } + } } # ====== BASE PREP: BASE IMAGE PREPARATION ====== @@ -360,7 +369,7 @@ function BasePrep { # TODO update to use proxy # Install OpenSSH if SSH enabled - $sshEnabled = [System.Convert]::ToBoolean("{{ WindowsSSHEnabled }}") + $sshEnabled=[System.Convert]::ToBoolean("{{WindowsSSHEnabled}}") if ( $sshEnabled ) { Install-OpenSSH -SSHKeys $SSHKeys } @@ -377,9 +386,13 @@ function BasePrep { Create-Directory -FullPath "c:\k" Write-Log "Remove `"NT AUTHORITY\Authenticated Users`" write permissions on files in c:\k" icacls.exe "c:\k" /inheritance:r + if ($LASTEXITCODE -ne 0) { throw "icacls.exe failed to set inheritance on c:\k (exit code $LASTEXITCODE)" } icacls.exe "c:\k" /grant:r SYSTEM:`(OI`)`(CI`)`(F`) + if ($LASTEXITCODE -ne 0) { throw "icacls.exe failed to grant SYSTEM permissions on c:\k (exit code $LASTEXITCODE)" } icacls.exe "c:\k" /grant:r BUILTIN\Administrators:`(OI`)`(CI`)`(F`) + if ($LASTEXITCODE -ne 0) { throw "icacls.exe failed to grant Administrators permissions on c:\k (exit code $LASTEXITCODE)" } icacls.exe "c:\k" /grant:r BUILTIN\Users:`(OI`)`(CI`)`(RX`) + if ($LASTEXITCODE -ne 0) { throw "icacls.exe failed to grant Users permissions on c:\k (exit code $LASTEXITCODE)" } Write-Log "c:\k permissions: " icacls.exe "c:\k" Get-ProvisioningScripts @@ -395,12 +408,13 @@ function BasePrep { # oras initialization, including install and login, must be in front of Install-CredentialProvider, Get-KubePackage and Install-Containerd-Based-On-Kubernetes-Version if ((Test-Path variable:global:BootstrapProfileContainerRegistryServer) -and - -not [string]::IsNullOrWhiteSpace($global:BootstrapProfileContainerRegistryServer)) { - # variable exists and is not empty/whitespace + -not [string]::IsNullOrWhiteSpace($global:BootstrapProfileContainerRegistryServer)) { + # variable exists and is not empty/whitespace if (Get-Command -Name Initialize-Oras -ErrorAction SilentlyContinue) { Logs-To-Event -TaskName "AKS.WindowsCSE.InitializeOras" -TaskMessage "Ensure oras is initialized for network isolated cluster" Initialize-Oras - } else { + } + else { Write-Log "Initialize-Oras is not a recognized function, will skip oras initialization for network isolated cluster" } } @@ -408,19 +422,20 @@ function BasePrep { # to ensure we don't introduce any incompatibility between base CSE + CSE package versions if (Get-Command -Name Install-SecureTLSBootstrapClient -ErrorAction SilentlyContinue) { Install-SecureTLSBootstrapClient -KubeDir $global:KubeDir -CustomSecureTLSBootstrapClientDownloadUrl $global:CustomSecureTLSBootstrappingClientDownloadURL - } else { + } + else { Write-Log "Install-SecureTLSBootstrapClient is not a recognized function, will skip installation of the secure TLS bootstrap client" } - Install-CredentialProvider -KubeDir $global:KubeDir -CustomCloudContainerRegistryDNSSuffix {{if IsAKSCustomCloud}}"{{ AKSCustomCloudContainerRegistryDNSSuffix }}"{{else}}""{{end}} + Install-CredentialProvider -KubeDir $global:KubeDir -CustomCloudContainerRegistryDNSSuffix {{if IsAKSCustomCloud}}"{{AKSCustomCloudContainerRegistryDNSSuffix}}" {{else}}"" {{end}} Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL - $cniBinPath = $global:AzureCNIBinDir - $cniConfigPath = $global:AzureCNIConfDir + $cniBinPath=$global:AzureCNIBinDir + $cniConfigPath=$global:AzureCNIConfDir if ($global:NetworkPlugin -eq "kubenet") { - $cniBinPath = $global:CNIPath - $cniConfigPath = $global:CNIConfigPath + $cniBinPath=$global:CNIPath + $cniConfigPath=$global:CNIConfigPath } Install-Containerd-Based-On-Kubernetes-Version -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath -KubeDir $global:KubeDir -KubernetesVersion $global:KubeBinariesVersion @@ -444,7 +459,7 @@ function BasePrep { Update-DefenderPreferences - $windowsVersion = Get-WindowsVersion + $windowsVersion=Get-WindowsVersion if ($windowsVersion -eq "1809" -or $windowsVersion -eq "ltsc2022") { Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Start to enable secure TLS protocols" try { @@ -530,11 +545,13 @@ function NodePrep { -UseInstanceMetadata $global:UseInstanceMetadata ` -LoadBalancerSku $global:LoadBalancerSku ` -ExcludeMasterFromStandardLB $global:ExcludeMasterFromStandardLB ` - -TargetEnvironment {{if IsAKSCustomCloud}}"AzureStackCloud"{{else}}$TargetEnvironment{{end}} + -TargetEnvironment {{if IsAKSCustomCloud}}"AzureStackCloud" {{else}}$TargetEnvironment {{end}} + # we borrow the logic of AzureStackCloud to achieve AKSCustomCloud. + # In case of AKSCustomCloud, customer cloud env will be loaded from azurestackcloud.json {{if IsAKSCustomCloud}} - $azureStackConfigFile = [io.path]::Combine($global:KubeDir, "azurestackcloud.json") - $envJSON = "{{ GetBase64EncodedEnvironmentJSON }}" + $azureStackConfigFile=[io.path]::Combine($global:KubeDir, "azurestackcloud.json") + $envJSON="{{GetBase64EncodedEnvironmentJSON}}" [io.file]::WriteAllBytes($azureStackConfigFile, [System.Convert]::FromBase64String($envJSON)) {{end}} @@ -543,13 +560,13 @@ function NodePrep { # newer versions accept -Location (optional) and -FailOnError. Bind only the # parameters the resolved function actually supports so a new VHD paired with # an older CSE zip does not fail with parameter-binding errors. - $getCACertsCmd = Get-Command -Name Get-CACertificates -ErrorAction Ignore - $getCACertsArgs = @{} + $getCACertsCmd=Get-Command -Name Get-CACertificates -ErrorAction Ignore + $getCACertsArgs=@{} if ($getCACertsCmd -and $getCACertsCmd.Parameters.ContainsKey('Location')) { - $getCACertsArgs['Location'] = $Location + $getCACertsArgs['Location']=$Location } if ($getCACertsCmd -and $getCACertsCmd.Parameters.ContainsKey('FailOnError')) { - $getCACertsArgs['FailOnError'] = $true + $getCACertsArgs['FailOnError']=$true } Get-CACertificates @getCACertsArgs @@ -592,22 +609,25 @@ function NodePrep { -AADClientSecret $([System.Text.Encoding]::ASCII.GetString([System.Convert]::FromBase64String($AADClientSecret))) ` -NetworkAPIVersion $NetworkAPIVersion ` -AzureEnvironmentFilePath $([io.path]::Combine($global:KubeDir, "azurestackcloud.json")) ` - -IdentitySystem "{{ GetIdentitySystem }}" + -IdentitySystem "{{GetIdentitySystem}}" } New-ExternalHnsNetwork -IsDualStackEnabled $global:IsDualStackEnabled # Turn off Firewall to enable pods to talk to service endpoints. (Kubelet should eventually do this) netsh advfirewall set allprofiles state off + if ($LASTEXITCODE -ne 0) { throw "netsh advfirewall failed to disable firewall (exit code $LASTEXITCODE)" } # To ensure we don't introduce any incompatibility between base CSE + CSE package versions if (Get-Command -Name Enable-WindowsCiliumNetworking -ErrorAction SilentlyContinue) { if ($global:EnableWindowsCiliumNetworking) { Enable-WindowsCiliumNetworking - } else { + } + else { Write-Log "Windows Cilium Networking is not enabled, will skip Windows Cilium Networking installation" } - } else { + } + else { Write-Log "Enable-WindowsCiliumNetworking is not a recognized function, will skip Windows Cilium Networking installation" } @@ -616,7 +636,7 @@ function NodePrep { } if ($global:TLSBootstrapToken -or $global:EnableSecureTLSBootstrapping) { Write-Log "Removing temporary kube config" - $kubeConfigFile = [io.path]::Combine($KubeDir, "config") + $kubeConfigFile=[io.path]::Combine($KubeDir, "config") Remove-Item $kubeConfigFile } @@ -638,8 +658,7 @@ function NodePrep { Start-InstallGPUDriver -EnableInstall $global:ConfigGPUDriverIfNeeded -GpuDriverURL $global:GpuDriverURL - if (Test-Path $CacheDir) - { + if (Test-Path $CacheDir) { Write-Log "Removing aks cache directory" Remove-Item $CacheDir -Recurse -Force } @@ -649,16 +668,19 @@ function NodePrep { if ($global:RebootNeeded) { Logs-To-Event -TaskName "AKS.WindowsCSE.RestartComputer" -TaskMessage "Setup Complete, calling Postpone-RestartComputer with reboot" Postpone-RestartComputer - } else { + } + else { Logs-To-Event -TaskName "AKS.WindowsCSE.StartScheduledTask" -TaskMessage "Setup Complete, start NodeResetScriptTask to register Windows node without reboot" Start-NodeResetScriptTask } + Write-Log "NodePrep completed successfully" Logs-To-Event -TaskName "AKS.WindowsCSE.NodePrep" -TaskMessage "NodePrep completed successfully" } -try -{ +try { + . Get-HelperScripts + Logs-To-Event -TaskName "AKS.WindowsCSE.ExecuteCustomDataSetupScript" -TaskMessage ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AADClientId $AADClientId -NetworkAPIVersion $NetworkAPIVersion -TargetEnvironment $TargetEnvironment -CSEResultFilePath $CSEResultFilePath" # Exit early if the script has been executed @@ -688,24 +710,24 @@ try # when nodes are created from that VHD image. if (-not (Test-Path "C:\AzureData\base_prep.complete")) { BasePrep - } else { + } + else { Write-Log "Skipping basePrep - base_prep.complete file exists" } if (-not $PreProvisionOnly) { NodePrep - } else { + } + else { Write-Log "Skipping nodePrep - pre-provision only mode" } } -catch -{ +catch { Resolve-Error # Set-ExitCode will exit with the specified ExitCode immediately and not be caught by this catch block # Ideally all exceptions will be handled and no exception will be thrown. Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_UNKNOWN -ErrorMessage $_ } -finally -{ +finally { # Generate CSE result so it can be returned as the CSE response in csecmd.ps1 $ExecutionDuration=$(New-Timespan -Start $StartTime -End $(Get-Date)) Write-Log "CSE ExecutionDuration: $ExecutionDuration. ExitCode: $global:ExitCode" @@ -713,22 +735,25 @@ finally Logs-To-Event -TaskName "AKS.WindowsCSE.cse_main" -TaskMessage "ExitCode: $global:ExitCode. ErrorMessage: $global:ErrorMessage." # Create appropriate completion file based on mode - $completionFilePath = if ($PreProvisionOnly) { "C:\AzureData\base_prep.complete" } else { $CSEResultFilePath } + $completionFilePath=if ($PreProvisionOnly) { "C:\AzureData\base_prep.complete" } else { $CSEResultFilePath } if ($global:ExitCode -eq 0) { Set-Content -Path $completionFilePath -Value $global:ExitCode -Force - } else { - # $JsonString = "ExitCode: |{0}|, Output: |{1}|, Error: |{2}|" + } + else { + # $JsonString="ExitCode: |{0}|, Output: |{1}|, Error: |{2}|" # Max length of the full error message returned by Windows CSE is ~256. We use 240 to be safe. - $errorMessageLength = "ExitCode: |$global:ExitCode|, Output: |$($global:ErrorCodeNames[$global:ExitCode])|, Error: ||".Length - $turncatedErrorMessage = $global:ErrorMessage.Substring(0, [Math]::Min(240 - $errorMessageLength, $global:ErrorMessage.Length)) + $errorMessageLength="ExitCode: |$global:ExitCode|, Output: |$($global:ErrorCodeNames[$global:ExitCode])|, Error: ||".Length + $turncatedErrorMessage=$global:ErrorMessage.Substring(0, [Math]::Min(240 - $errorMessageLength, $global:ErrorMessage.Length)) Set-Content -Path $completionFilePath -Value "ExitCode: |$global:ExitCode|, Output: |$($global:ErrorCodeNames[$global:ExitCode])|, Error: |$turncatedErrorMessage|" } - if ($global:ExitCode -eq $global:WINDOWS_CSE_ERROR_DOWNLOAD_CSE_PACKAGE) { - Write-Log "Do not call Upload-GuestVMLogs because there is no cse script package downloaded" + # Upload-GuestVMLogs is defined in the CSE scripts package (configfunc.ps1). + # If the CSE scripts package failed to download or load, the function will not be available. + if (Get-Command -Name Upload-GuestVMLogs -ErrorAction SilentlyContinue) { + Upload-GuestVMLogs -ExitCode $global:ExitCode } else { - Upload-GuestVMLogs -ExitCode $global:ExitCode + Write-Log "Upload-GuestVMLogs is not available, skipping log upload (CSE scripts package may not have been loaded)" } } diff --git a/parts/windows/windowscsehelper.ps1 b/parts/windows/windowscsehelper.ps1 index 2d54c6f8d5e..e7d77c12e6c 100644 --- a/parts/windows/windowscsehelper.ps1 +++ b/parts/windows/windowscsehelper.ps1 @@ -4,7 +4,7 @@ # Define all exit codes in Windows CSE # It must match `[A-Z_]+` $global:WINDOWS_CSE_SUCCESS=0 -$global:WINDOWS_CSE_ERROR_UNKNOWN=1 # For unexpected error caught by the catch block in kuberneteswindowssetup.ps1 +$global:WINDOWS_CSE_ERROR_UNKNOWN=1 # For unexpected error caught by the catch block in kuberneteswindowssetup.ps1.template $global:WINDOWS_CSE_ERROR_DOWNLOAD_FILE_WITH_RETRY=2 $global:WINDOWS_CSE_ERROR_INVOKE_EXECUTABLE=3 $global:WINDOWS_CSE_ERROR_FILE_NOT_EXIST=4 @@ -68,7 +68,7 @@ $global:WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_URL_NOT_EXE=61 $global:WINDOWS_CSE_ERROR_UPDATING_KUBE_CLUSTER_CONFIG=62 $global:WINDOWS_CSE_ERROR_GET_NODE_IPV6_IP=63 $global:WINDOWS_CSE_ERROR_GET_CONTAINERD_VERSION=64 -$global:WINDOWS_CSE_ERROR_INSTALL_CREDENTIAL_PROVIDER = 65 # exit code for installing credential provider +$global:WINDOWS_CSE_ERROR_INSTALL_CREDENTIAL_PROVIDER=65 # exit code for installing credential provider $global:WINDOWS_CSE_ERROR_DOWNLOAD_CREDEDNTIAL_PROVIDER=66 # exit code for downloading credential provider failure $global:WINDOWS_CSE_ERROR_CREDENTIAL_PROVIDER_CONFIG=67 # exit code for checking credential provider config failure $global:WINDOWS_CSE_ERROR_ADJUST_PAGEFILE_SIZE=68 @@ -93,7 +93,7 @@ $global:WINDOWS_CSE_ERROR_ORAS_PULL_CONTAINERD=84 # exit code for error pulling $global:WINDOWS_CSE_ERROR_MAX_CODE=85 # Please add new error code for downloading new packages in RP code too -$global:ErrorCodeNames = @( +$global:ErrorCodeNames=@( "WINDOWS_CSE_SUCCESS", "WINDOWS_CSE_ERROR_UNKNOWN", "WINDOWS_CSE_ERROR_DOWNLOAD_FILE_WITH_RETRY", @@ -182,33 +182,33 @@ $global:ErrorCodeNames = @( ) # The package domain to be used -$global:PackageDownloadFqdn = $null +$global:PackageDownloadFqdn=$null # The preferred package FQDN -$global:PreferredPackageDownloadFqdn = "packages.aks.azure.com" +$global:PreferredPackageDownloadFqdn="packages.aks.azure.com" # Fallback FQDN if preferred cannot be contacted -$global:FallbackPackageDownloadFqdn = "acs-mirror.azureedge.net" +$global:FallbackPackageDownloadFqdn="acs-mirror.azureedge.net" # NOTE: KubernetesVersion does not contain "v" -$global:MinimalKubernetesVersionWithLatestContainerd = "1.28.0" # Will change it to the correct version when we support new Windows containerd version +$global:MinimalKubernetesVersionWithLatestContainerd="1.28.0" # Will change it to the correct version when we support new Windows containerd version # The minimum kubernetes version to use containerd 2.x -$global:MinimalKubernetesVersionWithLatestContainerd2 = "1.33.0" +$global:MinimalKubernetesVersionWithLatestContainerd2="1.33.0" # Although the contianerd package url is set in AKS RP code now, we still need to update the following variables for AgentBaker Windows E2E tests. # Define containerd version template -$global:ContainerdPackageTemplate = "v{0}-azure.1/binaries/containerd-v{0}-azure.1-windows-amd64.tar.gz" +$global:ContainerdPackageTemplate="v{0}-azure.1/binaries/containerd-v{0}-azure.1-windows-amd64.tar.gz" # Version numbers only - used in various places -$global:StableContainerdVersion = "1.6.35" -$global:LatestContainerdVersion = "1.7.20" -$global:LatestContainerd2Version = "2.0.4" +$global:StableContainerdVersion="1.6.35" +$global:LatestContainerdVersion="1.7.20" +$global:LatestContainerd2Version="2.0.4" -$global:WindowsVersion2025 = "2025" +$global:WindowsVersion2025="2025" # Full package paths are generated using [string]::Format($global:ContainerdPackageTemplate, $version) when needed -$global:EventsLoggingDir = "C:\WindowsAzure\Logs\Plugins\Microsoft.Compute.CustomScriptExtension\Events\" -$global:TaskName = "" -$global:TaskTimeStamp = "" +$global:EventsLoggingDir="C:\WindowsAzure\Logs\Plugins\Microsoft.Compute.CustomScriptExtension\Events\" +$global:TaskName="" +$global:TaskTimeStamp="" # This filter removes null characters (\0) which are captured in nssm.exe output when logged through powershell filter RemoveNulls { $_ -replace '\0', '' } @@ -216,77 +216,75 @@ filter RemoveNulls { $_ -replace '\0', '' } filter Timestamp { "$(Get-Date -Format o): $_" } function Write-Log($message) { - $msg = $message | Timestamp - Write-Host $msg + $msg=$message | Timestamp + Write-Host $msg } function DownloadFileOverHttp { Param( - [Parameter(Mandatory = $true)][string] + [Parameter(Mandatory=$true)][string] $Url, - [Parameter(Mandatory = $true)][string] + [Parameter(Mandatory=$true)][string] $DestinationPath, - [Parameter(Mandatory = $true)][int] + [Parameter(Mandatory=$true)][int] $ExitCode ) # First check to see if a file with the same name is already cached on the VHD - $cleanUrl = $Url.Split('?')[0] - $fileName = [IO.Path]::GetFileName($cleanUrl) + $cleanUrl=$Url.Split('?')[0] + $fileName=[IO.Path]::GetFileName($cleanUrl) - $search = @() + $search=@() if ($global:CacheDir -and (Test-Path $global:CacheDir)) { - $search = [IO.Directory]::GetFiles($global:CacheDir, $fileName, [IO.SearchOption]::AllDirectories) + $search=[IO.Directory]::GetFiles($global:CacheDir, $fileName, [IO.SearchOption]::AllDirectories) } if ($search.Count -ne 0) { Write-Log "Using cached version of $fileName - Copying file from $($search[0]) to $DestinationPath" Copy-Item -Path $search[0] -Destination $DestinationPath -Force - } - else { - $secureProtocols = @() - $insecureProtocols = @([System.Net.SecurityProtocolType]::SystemDefault, [System.Net.SecurityProtocolType]::Ssl3) + } else { + $secureProtocols=@() + $insecureProtocols=@([System.Net.SecurityProtocolType]::SystemDefault, [System.Net.SecurityProtocolType]::Ssl3) foreach ($protocol in [System.Enum]::GetValues([System.Net.SecurityProtocolType])) { if ($insecureProtocols -notcontains $protocol) { $secureProtocols += $protocol } } - [System.Net.ServicePointManager]::SecurityProtocol = $secureProtocols + [System.Net.ServicePointManager]::SecurityProtocol=$secureProtocols - $MappedUrl = Update-BaseUrl -InitialUrl $Url + $MappedUrl=Update-BaseUrl -InitialUrl $Url Write-Log "Updated URL $Url -> $MappedUrl to download $fileName to $DestinationPath" - $oldProgressPreference = $ProgressPreference - $ProgressPreference = 'SilentlyContinue' + $oldProgressPreference=$ProgressPreference + $ProgressPreference='SilentlyContinue' - $downloadTimer = [System.Diagnostics.Stopwatch]::StartNew() + $downloadTimer=[System.Diagnostics.Stopwatch]::StartNew() try { - $args = @{Uri=$MappedUrl; Method="Get"; OutFile=$DestinationPath; ErrorAction="Stop"} - Retry-Command -Command "Invoke-RestMethod" -Args $args -Retries 5 -RetryDelaySeconds 10 + $arglist=@{Uri=$MappedUrl; Method="Get"; OutFile=$DestinationPath; ErrorAction="Stop" } + Retry-Command -Command "Invoke-RestMethod" -Args $arglist -Retries 5 -RetryDelaySeconds 10 } catch { Set-ExitCode -ExitCode $ExitCode -ErrorMessage "Failed in downloading $MappedUrl. Error: $_" } $downloadTimer.Stop() - $elapsedMs = $downloadTimer.ElapsedMilliseconds - - if ($global:AppInsightsClient -ne $null) { - $event = New-Object "Microsoft.ApplicationInsights.DataContracts.EventTelemetry" - $event.Name = "FileDownload" - $event.Properties["FileName"] = $fileName - $event.Metrics["DurationMs"] = $elapsedMs - $global:AppInsightsClient.TrackEvent($event) + $elapsedMs=$downloadTimer.ElapsedMilliseconds + + if ($null -ne $global:AppInsightsClient) { + $evt=New-Object "Microsoft.ApplicationInsights.DataContracts.EventTelemetry" + $evt.Name="FileDownload" + $evt.Properties["FileName"]=$fileName + $evt.Metrics["DurationMs"]=$elapsedMs + $global:AppInsightsClient.TrackEvent($evt) } - $ProgressPreference = $oldProgressPreference + $ProgressPreference=$oldProgressPreference Write-Log "Downloaded file $MappedUrl to $DestinationPath in $elapsedMs ms" Get-Item $DestinationPath -ErrorAction Continue | Format-List | Out-String | Write-Log } } -function Set-ExitCode -{ +function Set-ExitCode { Param( [Parameter(Mandatory=$true)][int] $ExitCode, @@ -300,24 +298,23 @@ function Set-ExitCode exit $ExitCode } -function Start-NodeResetScriptTask -{ +function Start-NodeResetScriptTask { Param( [Parameter(Mandatory=$false)][int] - $TimeoutSeconds = 180 + $TimeoutSeconds=180 ) - $taskName = "k8s-restart-job" - $taskRunningResult = 0x00041301 - $previousRunTime = (Get-ScheduledTaskInfo -TaskName $taskName).LastRunTime + $taskName="k8s-restart-job" + $taskRunningResult=0x00041301 + $previousRunTime=(Get-ScheduledTaskInfo -TaskName $taskName).LastRunTime Start-ScheduledTask -TaskName $taskName - $timer = [Diagnostics.Stopwatch]::StartNew() + $timer=[Diagnostics.Stopwatch]::StartNew() do { - $taskInfo = Get-ScheduledTaskInfo -TaskName $taskName - $task = Get-ScheduledTask -TaskName $taskName + $taskInfo=Get-ScheduledTaskInfo -TaskName $taskName + $task=Get-ScheduledTask -TaskName $taskName if ($task.State -eq "Ready" -and $taskInfo.LastRunTime -ne $previousRunTime) { - $taskInfo = Get-ScheduledTaskInfo -TaskName $taskName + $taskInfo=Get-ScheduledTaskInfo -TaskName $taskName if ($taskInfo.LastRunTime -ne $previousRunTime -and $taskInfo.LastTaskResult -ne $taskRunningResult) { break } @@ -336,7 +333,7 @@ function Start-NodeResetScriptTask Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_START_NODE_RESET_SCRIPT_TASK -ErrorMessage "NodeResetScriptTask failed with result $($taskInfo.LastTaskResult)" } - $kubeletService = Get-Service -Name "kubelet" -ErrorAction SilentlyContinue + $kubeletService=Get-Service -Name "kubelet" -ErrorAction SilentlyContinue if ($null -eq $kubeletService -or $kubeletService.Status -ne "Running") { Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_START_NODE_RESET_SCRIPT_TASK -ErrorMessage "kubelet service is not running after NodeResetScriptTask completed" } @@ -344,25 +341,23 @@ function Start-NodeResetScriptTask Write-Log -Message "We waited [$($timer.Elapsed.TotalSeconds)] seconds on NodeResetScriptTask" } -function Postpone-RestartComputer -{ +function Postpone-RestartComputer { Logs-To-Event -TaskName "AKS.WindowsCSE.PostponeRestartComputer" -TaskMessage "Start to create an one-time task to restart the VM" - $action = New-ScheduledTaskAction -Execute "powershell.exe" -Argument " -Command `"Restart-Computer -Force`"" - $principal = New-ScheduledTaskPrincipal -UserId SYSTEM -LogonType ServiceAccount -RunLevel Highest + $action=New-ScheduledTaskAction -Execute "powershell.exe" -Argument " -Command `"Restart-Computer -Force`"" + $principal=New-ScheduledTaskPrincipal -UserId SYSTEM -LogonType ServiceAccount -RunLevel Highest # trigger this task once - $trigger = New-JobTrigger -At (Get-Date).AddSeconds(15).DateTime -Once - $definition = New-ScheduledTask -Action $action -Principal $principal -Trigger $trigger -Description "Restart computer after provisioning the VM" + $trigger=New-JobTrigger -At (Get-Date).AddSeconds(15).DateTime -Once + $definition=New-ScheduledTask -Action $action -Principal $principal -Trigger $trigger -Description "Restart computer after provisioning the VM" Register-ScheduledTask -TaskName "restart-computer" -InputObject $definition Write-Log "Created an one-time task to restart the VM" } -function Create-Directory -{ +function Create-Directory { Param( [Parameter(Mandatory=$true)][string] $FullPath, [Parameter(Mandatory=$false)][string] - $DirectoryUsage = "general purpose" + $DirectoryUsage="general purpose" ) if (-Not (Test-Path $FullPath)) { @@ -375,19 +370,19 @@ function Create-Directory # https://stackoverflow.com/a/34559554/697126 function New-TemporaryDirectory { - $parent = [System.IO.Path]::GetTempPath() - [string] $name = [System.Guid]::NewGuid() + $parent=[System.IO.Path]::GetTempPath() + [string] $name=[System.Guid]::NewGuid() New-Item -ItemType Directory -Path (Join-Path $parent $name) } function AKS-Expand-Archive { Param( - [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][string]$Path, - [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][string]$DestinationPath, - [Parameter(Mandatory = $false)][ValidateNotNullOrEmpty()][boolean]$Force + [Parameter(Mandatory=$true)][ValidateNotNullOrEmpty()][string]$Path, + [Parameter(Mandatory=$true)][ValidateNotNullOrEmpty()][string]$DestinationPath, + [Parameter(Mandatory=$false)][ValidateNotNullOrEmpty()][boolean]$Force ) - try { + try { Expand-Archive -Path $Path -DestinationPath ${DestinationPath} -ErrorAction Stop -Force Write-Log "Successfully expanded file $Path to $DestinationPath" } catch { @@ -399,23 +394,22 @@ function AKS-Expand-Archive { function Retry-Command { Param( - [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][string] + [Parameter(Mandatory=$true)][ValidateNotNullOrEmpty()][string] $Command, - [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][hashtable] + [Parameter(Mandatory=$true)][ValidateNotNullOrEmpty()][hashtable] $Args, - [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][int] + [Parameter(Mandatory=$true)][ValidateNotNullOrEmpty()][int] $Retries, - [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][int] + [Parameter(Mandatory=$true)][ValidateNotNullOrEmpty()][int] $RetryDelaySeconds ) - for ($i = 0; ; ) { + for ($i=0; ; ) { try { # Do not log Args since Args may contain sensitive data Write-Log "Retry $i : $command" return & $Command @Args - } - catch { + } catch { $i++ if ($i -ge $Retries) { throw $_ @@ -434,22 +428,21 @@ function Invoke-Executable { [Parameter(Mandatory=$true)][int] $ExitCode, [int[]] - $AllowedExitCodes = @(0), + $AllowedExitCodes=@(0), [int] - $Retries = 0, + $Retries=0, [int] - $RetryDelaySeconds = 1 + $RetryDelaySeconds=1 ) - for ($i = 0; $i -le $Retries; $i++) { + for ($i=0; $i -le $Retries; $i++) { Write-Log "$i - Running $Executable $ArgList ..." & $Executable $ArgList if ($LASTEXITCODE -notin $AllowedExitCodes) { Write-Log "$Executable returned unsuccessfully with exit code $LASTEXITCODE" Start-Sleep -Seconds $RetryDelaySeconds continue - } - else { + } else { Write-Log "$Executable returned successfully" return } @@ -460,9 +453,9 @@ function Invoke-Executable { function Assert-FileExists { Param( - [Parameter(Mandatory = $true)][string] + [Parameter(Mandatory=$true)][string] $Filename, - [Parameter(Mandatory = $true)][int] + [Parameter(Mandatory=$true)][int] $ExitCode ) @@ -476,12 +469,12 @@ function Get-WindowsBuildNumber { } function Get-WindowsVersion { - $buildNumber = Get-WindowsBuildNumber + $buildNumber=Get-WindowsBuildNumber switch ($buildNumber) { "17763" { return "1809" } "20348" { return "ltsc2022" } "25398" { return "23H2" } - {$_ -ge "25399" -and $_ -le "30397"} { return $global:WindowsVersion2025 } + { $_ -ge "25399" -and $_ -le "30397" } { return $global:WindowsVersion2025 } Default { Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_NOT_FOUND_BUILD_NUMBER -ErrorMessage "Failed to find the windows build number: $buildNumber" } @@ -489,12 +482,12 @@ function Get-WindowsVersion { } function Get-WindowsPauseVersion { - $buildNumber = Get-WindowsBuildNumber + $buildNumber=Get-WindowsBuildNumber switch ($buildNumber) { "17763" { return "1809" } "20348" { return "ltsc2022" } "25398" { return "ltsc2022" } - {$_ -ge "25399" -and $_ -le "30397"} { return "ltsc2022" } + { $_ -ge "25399" -and $_ -le "30397" } { return "ltsc2022" } Default { Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_NOT_FOUND_BUILD_NUMBER -ErrorMessage "Failed to find the windows build number: $buildNumber" } @@ -502,65 +495,65 @@ function Get-WindowsPauseVersion { } function Install-Containerd-Based-On-Kubernetes-Version { - Param( - [Parameter(Mandatory = $true)][string] - $ContainerdUrl, - [Parameter(Mandatory = $true)][string] - $CNIBinDir, - [Parameter(Mandatory = $true)][string] - $CNIConfDir, - [Parameter(Mandatory = $true)][string] - $KubeDir, - [Parameter(Mandatory = $true)][string] - $KubernetesVersion - ) - - # Get the current Windows version, this is interim since we are progressively supporting containerd 2.0 for all Windows version. for now only test2025 - $windowsVersion = Get-WindowsVersion - Write-Log "Install Containerd with ContainerdURL: $ContainerdUrl, KubernetesVersion: $KubernetesVersion, WindowsVersion: $windowsVersion" - Logs-To-Event -TaskName "AKS.WindowsCSE.InstallContainerdBasedOnKubernetesVersion" -TaskMessage "Start to install ContainerD based on kubernetes version. ContainerdUrl: $global:ContainerdUrl, KubernetesVersion: $global:KubeBinariesVersion, Windows Version: $windowsVersion" - - # $global:ContainerdUrl is set from RP ContainerService.properties.orchestratorProfile.KubernetesConfig.WindowsContainerdURL - # it can be - # - a full URL. e.g., "https://packages.aks.azure.com/containerd/windows/v0.0.46/binaries/containerd-v0.0.46-windows-amd64.tar.gz" - # - an endpoint: e.g., "https://packages.aks.azure.com/containerd/windows/" - - # We only set containerd package based on kubernetes version when $global:ContainerdUrl ends with "/" so we support: - # 1. Current behavior to set the full URL - # 2. Setting containerd package in toggle for test purpose or hotfix - - $containerdVersion=$global:StableContainerdVersion - Write-Log "Install Containerd with request URL : $ContainerdUrl, Kubernetes version: $KubernetesVersion, Windows version: $windowsVersion." - - if ($ContainerdUrl.EndsWith("/")) { - # for now we only preview containerd 2.0 for Windows 2025 - if ($windowsVersion -eq $global:WindowsVersion2025) { - $containerdVersion=$global:LatestContainerd2Version - } elseif (([version]$KubernetesVersion).CompareTo([version]$global:MinimalKubernetesVersionWithLatestContainerd) -ge 0) { - $containerdVersion=$global:LatestContainerdVersion - } - $containerdPackage = [string]::Format($global:ContainerdPackageTemplate, $containerdVersion) - $ContainerdUrl = $ContainerdUrl + $containerdPackage - } elseif ( $windowsVersion -eq $global:WindowsVersion2025) { - # TODO (beileihuang) : remove this else if block when RP is release to set the correct versions for 2025 - $containerdPattern = "v\d+\.\d+\.\d+-azure\.\d+/binaries/containerd-v\d+\.\d+\.\d+-azure\.\d+-windows-amd64\.tar\.gz" - if ($ContainerdUrl -match $containerdPattern) { - $matchedPath = $matches[0] - $containerd2Package = [string]::Format($global:ContainerdPackageTemplate, $global:LatestContainerd2Version) - $ContainerdUrl = $ContainerdUrl.Replace($matchedPath, $containerd2Package) + Param( + [Parameter(Mandatory=$true)][string] + $ContainerdUrl, + [Parameter(Mandatory=$true)][string] + $CNIBinDir, + [Parameter(Mandatory=$true)][string] + $CNIConfDir, + [Parameter(Mandatory=$true)][string] + $KubeDir, + [Parameter(Mandatory=$true)][string] + $KubernetesVersion + ) + + # Get the current Windows version, this is interim since we are progressively supporting containerd 2.0 for all Windows version. for now only test2025 + $windowsVersion=Get-WindowsVersion + Write-Log "Install Containerd with ContainerdURL: $ContainerdUrl, KubernetesVersion: $KubernetesVersion, WindowsVersion: $windowsVersion" + Logs-To-Event -TaskName "AKS.WindowsCSE.InstallContainerdBasedOnKubernetesVersion" -TaskMessage "Start to install ContainerD based on kubernetes version. ContainerdUrl: $global:ContainerdUrl, KubernetesVersion: $global:KubeBinariesVersion, Windows Version: $windowsVersion" + + # $global:ContainerdUrl is set from RP ContainerService.properties.orchestratorProfile.KubernetesConfig.WindowsContainerdURL + # it can be + # - a full URL. e.g., "https://packages.aks.azure.com/containerd/windows/v0.0.46/binaries/containerd-v0.0.46-windows-amd64.tar.gz" + # - an endpoint: e.g., "https://packages.aks.azure.com/containerd/windows/" + + # We only set containerd package based on kubernetes version when $global:ContainerdUrl ends with "/" so we support: + # 1. Current behavior to set the full URL + # 2. Setting containerd package in toggle for test purpose or hotfix + + $containerdVersion=$global:StableContainerdVersion + Write-Log "Install Containerd with request URL : $ContainerdUrl, Kubernetes version: $KubernetesVersion, Windows version: $windowsVersion." + + if ($ContainerdUrl.EndsWith("/")) { + # for now we only preview containerd 2.0 for Windows 2025 + if ($windowsVersion -eq $global:WindowsVersion2025) { + $containerdVersion=$global:LatestContainerd2Version + } elseif (([version]$KubernetesVersion).CompareTo([version]$global:MinimalKubernetesVersionWithLatestContainerd) -ge 0) { + $containerdVersion=$global:LatestContainerdVersion + } + $containerdPackage=[string]::Format($global:ContainerdPackageTemplate, $containerdVersion) + $ContainerdUrl=$ContainerdUrl + $containerdPackage + } elseif ( $windowsVersion -eq $global:WindowsVersion2025) { + # TODO (beileihuang) : remove this else if block when RP is release to set the correct versions for 2025 + $containerdPattern="v\d+\.\d+\.\d+-azure\.\d+/binaries/containerd-v\d+\.\d+\.\d+-azure\.\d+-windows-amd64\.tar\.gz" + if ($ContainerdUrl -match $containerdPattern) { + $matchedPath=$matches[0] + $containerd2Package=[string]::Format($global:ContainerdPackageTemplate, $global:LatestContainerd2Version) + $ContainerdUrl=$ContainerdUrl.Replace($matchedPath, $containerd2Package) + } } - } - Write-Log "Install Containerd with resolved containerd pacakge url: $ContainerdUrl, Kubernetes version: $KubernetesVersion, Windows version: $windowsVersion." - Logs-To-Event -TaskName "AKS.WindowsCSE.InstallContainerd" -TaskMessage "Start to install ContainerD. ContainerdUrl: $ContainerdUrl" - Install-Containerd -ContainerdUrl $ContainerdUrl -CNIBinDir $CNIBinDir -CNIConfDir $CNIConfDir -KubeDir $KubeDir + Write-Log "Install Containerd with resolved containerd pacakge url: $ContainerdUrl, Kubernetes version: $KubernetesVersion, Windows version: $windowsVersion." + Logs-To-Event -TaskName "AKS.WindowsCSE.InstallContainerd" -TaskMessage "Start to install ContainerD. ContainerdUrl: $ContainerdUrl" + Install-Containerd -ContainerdUrl $ContainerdUrl -CNIBinDir $CNIBinDir -CNIConfDir $CNIConfDir -KubeDir $KubeDir } function Logs-To-Event { Param( - [Parameter(Mandatory = $true)][string] + [Parameter(Mandatory=$true)][string] $TaskName, - [Parameter(Mandatory = $true)][string] + [Parameter(Mandatory=$true)][string] $TaskMessage ) $eventLevel="Informational" @@ -571,19 +564,19 @@ function Logs-To-Event { $eventsFileName=[DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() $currentTime=$(Get-Date -Format "yyyy-MM-dd HH:mm:ss.fff") - $lastTaskName = "" - $lastTaskDuration = 0 + $lastTaskName="" + $lastTaskDuration=0 if ($global:TaskTimeStamp -ne "") { - $lastTaskName = $global:TaskName - $lastTaskDuration = $(New-Timespan -Start $global:TaskTimeStamp -End $currentTime) + $lastTaskName=$global:TaskName + $lastTaskDuration=$(New-Timespan -Start $global:TaskTimeStamp -End $currentTime) } - $global:TaskName = $TaskName - $global:TaskTimeStamp = $currentTime + $global:TaskName=$TaskName + $global:TaskTimeStamp=$currentTime Write-Log "$global:TaskName - $TaskMessage" - $TaskMessage = (echo $TaskMessage | ConvertTo-Json) - $messageJson = @" + $TaskMessage=(Write-Output $TaskMessage | ConvertTo-Json) + $messageJson=@" { "HostName": "$env:computername", "LastTaskName": "$lastTaskName", @@ -591,9 +584,9 @@ function Logs-To-Event { "CurrentTaskMessage": $TaskMessage } "@ - $messageJson = (echo $messageJson | ConvertTo-Json) + $messageJson=(Write-Output $messageJson | ConvertTo-Json) - $jsonString = @" + $jsonString=@" { "Timestamp": "$global:TaskTimeStamp", "OperationId": "$global:OperationId", @@ -614,46 +607,46 @@ function Logs-To-Event { # It will attempt to use the preferred FQDN first and if that fails it will fallback to the old CDN URL function Resolve-PackagesDownloadFqdn { Param( - [Parameter(Mandatory = $true)][string] + [Parameter(Mandatory=$true)][string] $PreferredFqdn, - [Parameter(Mandatory = $true)][string] + [Parameter(Mandatory=$true)][string] $FallbackFqdn, - [Parameter(Mandatory = $false)][int] - $Retries = 5, - [Parameter(Mandatory = $false)][int] - $WaitSleepSeconds = 1 + [Parameter(Mandatory=$false)][int] + $Retries=5, + [Parameter(Mandatory=$false)][int] + $WaitSleepSeconds=1 ) - $packageDownloadBaseUrl = $PreferredFqdn + $packageDownloadBaseUrl=$PreferredFqdn - for ($i = 1; $i -le $Retries; $i++) { + for ($i=1; $i -le $Retries; $i++) { # Confirm that we can establish connectivity to packages.aks.azure.com before node provisioning starts try { - $response = Invoke-WebRequest -Uri "https://${PreferredFqdn}/acs-mirror/healthz" -UseBasicParsing -TimeoutSec 5 -ErrorAction SilentlyContinue - $responseCode = [int]$response.StatusCode + $response=Invoke-WebRequest -Uri "https://${PreferredFqdn}/acs-mirror/healthz" -UseBasicParsing -TimeoutSec 5 -ErrorAction SilentlyContinue + $responseCode=[int]$response.StatusCode if ($responseCode -eq 200) { Write-Log "Established connectivity to $PreferredFqdn." | Out-Null break } } catch { - $responseCode = 0 + $responseCode=0 Write-Log "Exception while trying to establish connectivity to $PreferredFqdn. Exception: $_" | Out-Null if ($_.Exception.Response) { - $responseCode = [int]$_.Exception.Response.StatusCode + $responseCode=[int]$_.Exception.Response.StatusCode } } if ($i -eq $Retries) { # If we cannot establish connectivity to packages.aks.azure.com, fallback to old CDN URL - $packageDownloadBaseUrl = $FallbackFqdn + $packageDownloadBaseUrl=$FallbackFqdn break } else { Start-Sleep -Seconds $WaitSleepSeconds } } - $global:PackageDownloadFqdn = $packageDownloadBaseUrl + $global:PackageDownloadFqdn=$packageDownloadBaseUrl Logs-To-Event -TaskName "AKS.WindowsCSE.ResolvedPackageDomain" -TaskMessage "Package download FQDN: $global:PackageDownloadFqdn" } @@ -661,39 +654,38 @@ function Resolve-PackagesDownloadFqdn { # This function will swap the domain in the URL based on the verified package download FQDN function Update-BaseUrl { Param( - [Parameter(Mandatory = $true)][string] + [Parameter(Mandatory=$true)][string] $InitialUrl ) - $updatedUrl = $InitialUrl + $updatedUrl=$InitialUrl if (!($InitialUrl -match "acs-mirror\.azureedge\.net|packages\.aks\.azure\.com")) { # We're probably not in Public cloud return $updatedUrl } - if ($global:PackageDownloadFqdn -eq $null) { + if ($null -eq $global:PackageDownloadFqdn) { # We're in public cloud, but we haven't set the package download FQDN yet - $null = Resolve-PackagesDownloadFqdn -PreferredFqdn $global:PreferredPackageDownloadFqdn -FallbackFqdn $global:FallbackPackageDownloadFqdn + $null=Resolve-PackagesDownloadFqdn -PreferredFqdn $global:PreferredPackageDownloadFqdn -FallbackFqdn $global:FallbackPackageDownloadFqdn } # Replace domain based on the current package download FQDN if (($global:PackageDownloadFqdn -eq "packages.aks.azure.com") -and ($InitialUrl -like "https://acs-mirror.azureedge.net/*")) { - $updatedUrl = $InitialUrl -replace "acs-mirror.azureedge.net", $global:PackageDownloadFqdn + $updatedUrl=$InitialUrl -replace "acs-mirror.azureedge.net", $global:PackageDownloadFqdn } elseif (($global:PackageDownloadFqdn -eq "acs-mirror.azureedge.net") -and ($InitialUrl -like "https://packages.aks.azure.com/*")) { - $updatedUrl = $InitialUrl -replace "packages.aks.azure.com", $global:PackageDownloadFqdn + $updatedUrl=$InitialUrl -replace "packages.aks.azure.com", $global:PackageDownloadFqdn } return $updatedUrl } -function Resolve-Error ($ErrorRecord=$Error[0]) -{ - $ErrorRecord | Format-List * -Force - $ErrorRecord.InvocationInfo |Format-List * - $Exception = $ErrorRecord.Exception - for ($i = 0; $Exception; $i++, ($Exception = $Exception.InnerException)) - { "$i" * 80 - $Exception |Format-List * -Force - } +function Resolve-Error ($ErrorRecord=$Error[0]) { + $ErrorRecord | Format-List * -Force + $ErrorRecord.InvocationInfo | Format-List * + $Exception=$ErrorRecord.Exception + for ($i=0; $Exception; $i++, ($Exception=$Exception.InnerException)) { + "$i" * 80 + $Exception | Format-List * -Force + } } diff --git a/pkg/agent/baker.go b/pkg/agent/baker.go index 6fbb0a4ef86..0f7899ebe60 100644 --- a/pkg/agent/baker.go +++ b/pkg/agent/baker.go @@ -12,6 +12,7 @@ import ( "fmt" "reflect" "regexp" + "slices" "sort" "strconv" "strings" @@ -723,8 +724,17 @@ func normalizeResourceGroupNameForLabel(resourceGroupName string) string { // ValidateAndSetLinuxNodeBootstrappingConfiguration is exported only for temporary usage in e2e testing of new config. func ValidateAndSetLinuxNodeBootstrappingConfiguration(config *datamodel.NodeBootstrappingConfiguration) { + _ = ValidateAndSetLinuxNodeBootstrappingConfigurationWithError(config) +} + +// ValidateAndSetLinuxNodeBootstrappingConfigurationWithError validates and updates Linux node bootstrapping configuration. +func ValidateAndSetLinuxNodeBootstrappingConfigurationWithError(config *datamodel.NodeBootstrappingConfiguration) error { + if err := validateCustomLinuxOSConfig(config.AgentPoolProfile.GetCustomLinuxOSConfig()); err != nil { + return err + } + if config.KubeletConfig == nil { - return + return nil } kubeletFlags := config.KubeletConfig @@ -785,6 +795,39 @@ func ValidateAndSetLinuxNodeBootstrappingConfiguration(config *datamodel.NodeBoo if IsKubernetesVersionGe(config.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion, "1.34.0") { delete(kubeletFlags, "--streaming-connection-idle-timeout") } + return nil +} + +func validateCustomLinuxOSConfig(config *datamodel.CustomLinuxOSConfig) error { + if config == nil { + return nil + } + + if err := validateTransparentHugePageConfigValue( + "transparentHugePageEnabled", + config.TransparentHugePageEnabled, + []string{"always", "madvise", "never"}, + ); err != nil { + return err + } + + return validateTransparentHugePageConfigValue( + "transparentHugePageDefrag", + config.TransparentHugePageDefrag, + []string{"always", "defer", "defer+madvise", "madvise", "never"}, + ) +} + +func validateTransparentHugePageConfigValue(fieldName, value string, allowedValues []string) error { + if value == "" { + return nil + } + + if slices.Contains(allowedValues, value) { + return nil + } + + return fmt.Errorf("customLinuxOSConfig.%s value %q is invalid; allowed values are: %s", fieldName, value, strings.Join(allowedValues, ", ")) } func validateAndSetWindowsNodeBootstrappingConfiguration(config *datamodel.NodeBootstrappingConfiguration) { @@ -1890,27 +1933,12 @@ const ( containerdV1ConfigTemplate ContainerdConfigTemplate = `version = 2 oom_score = -999{{if HasDataDir }} root = "{{GetDataDir}}"{{- end}} -{{- if IsKata }} -[plugins."io.containerd.snapshotter.v1.erofs"] - default_size = "10G" - enable_fsverity = false - ovl_mount_options = [] - max_unmerged_layers = 1 - -[plugins."io.containerd.service.v1.diff-service"] - default = ["erofs", "walking"] - -[plugins."io.containerd.differ.v1.erofs"] - mkfs_options = ["-T0", "--mkfs-time", "--sort=none"] - enable_tar_index = false -{{- end}} [plugins."io.containerd.grpc.v1.cri"] sandbox_image = "{{GetPodInfraContainerSpec}}" enable_cdi = true [plugins."io.containerd.grpc.v1.cri".containerd] {{- if IsKata }} disable_snapshot_annotations = false - snapshotter = "overlayfs" {{- end}} {{- if IsArtifactStreamingEnabled }} snapshotter = "overlaybd" @@ -1967,15 +1995,8 @@ root = "{{GetDataDir}}"{{- end}} [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata] runtime_type = "io.containerd.kata.v2" privileged_without_host_devices = true - snapshotter = "overlayfs" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options] ConfigPath = "/usr/share/defaults/kata-containers/configuration.toml" -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview] - runtime_type = "io.containerd.kata.v2" - privileged_without_host_devices = true - snapshotter = "erofs" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview.options] - ConfigPath = "/usr/share/defaults/kata-containers/configuration-clh-templating.toml" [proxy_plugins] [proxy_plugins.tardev] type = "snapshot" @@ -1992,20 +2013,6 @@ root = "{{GetDataDir}}"{{- end}} containerdV2ConfigTemplate ContainerdConfigTemplate = `version = 2 oom_score = -999{{if HasDataDir }} root = "{{GetDataDir}}"{{- end}} -{{- if IsKata }} -[plugins."io.containerd.snapshotter.v1.erofs"] - default_size = "10G" - enable_fsverity = false - ovl_mount_options = [] - max_unmerged_layers = 1 - -[plugins."io.containerd.service.v1.diff-service"] - default = ["erofs", "walking"] - -[plugins."io.containerd.differ.v1.erofs"] - mkfs_options = ["-T0", "--mkfs-time", "--sort=none"] - enable_tar_index = false -{{- end}} [plugins."io.containerd.cri.v1.images"] {{- if IsArtifactStreamingEnabled }} snapshotter = "overlaybd" @@ -2061,15 +2068,8 @@ root = "{{GetDataDir}}"{{- end}} [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata] runtime_type = "io.containerd.kata.v2" privileged_without_host_devices = true - snapshotter = "overlayfs" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options] ConfigPath = "/usr/share/defaults/kata-containers/configuration.toml" -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview] - runtime_type = "io.containerd.kata.v2" - privileged_without_host_devices = true - snapshotter = "erofs" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview.options] - ConfigPath = "/usr/share/defaults/kata-containers/configuration-clh-templating.toml" [proxy_plugins] [proxy_plugins.tardev] type = "snapshot" @@ -2086,20 +2086,6 @@ root = "{{GetDataDir}}"{{- end}} containerdV2NoGPUConfigTemplate ContainerdConfigTemplate = `version = 2 oom_score = -999{{if HasDataDir }} root = "{{GetDataDir}}"{{- end}} -{{- if IsKata }} -[plugins."io.containerd.snapshotter.v1.erofs"] - default_size = "10G" - enable_fsverity = false - ovl_mount_options = [] - max_unmerged_layers = 1 - -[plugins."io.containerd.service.v1.diff-service"] - default = ["erofs", "walking"] - -[plugins."io.containerd.differ.v1.erofs"] - mkfs_options = ["-T0", "--mkfs-time", "--sort=none"] - enable_tar_index = false -{{- end}} [plugins."io.containerd.cri.v1.images"] {{- if IsArtifactStreamingEnabled }} snapshotter = "overlaybd" @@ -2142,15 +2128,8 @@ root = "{{GetDataDir}}"{{- end}} [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata] runtime_type = "io.containerd.kata.v2" privileged_without_host_devices = true - snapshotter = "overlayfs" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options] ConfigPath = "/usr/share/defaults/kata-containers/configuration.toml" -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview] - runtime_type = "io.containerd.kata.v2" - privileged_without_host_devices = true - snapshotter = "erofs" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview.options] - ConfigPath = "/usr/share/defaults/kata-containers/configuration-clh-templating.toml" [proxy_plugins] [proxy_plugins.tardev] type = "snapshot" @@ -2160,26 +2139,11 @@ root = "{{GetDataDir}}"{{- end}} containerdV1NoGPUConfigTemplate ContainerdConfigTemplate = `version = 2 oom_score = -999{{if HasDataDir }} root = "{{GetDataDir}}"{{- end}} -{{- if IsKata }} -[plugins."io.containerd.snapshotter.v1.erofs"] - default_size = "10G" - enable_fsverity = false - ovl_mount_options = [] - max_unmerged_layers = 1 - -[plugins."io.containerd.service.v1.diff-service"] - default = ["erofs", "walking"] - -[plugins."io.containerd.differ.v1.erofs"] - mkfs_options = ["-T0", "--mkfs-time", "--sort=none"] - enable_tar_index = false -{{- end}} [plugins."io.containerd.grpc.v1.cri"] sandbox_image = "{{GetPodInfraContainerSpec}}" [plugins."io.containerd.grpc.v1.cri".containerd] {{- if IsKata }} disable_snapshot_annotations = false - snapshotter = "overlayfs" {{- end}} {{- if IsArtifactStreamingEnabled }} snapshotter = "overlaybd" @@ -2221,15 +2185,8 @@ root = "{{GetDataDir}}"{{- end}} [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata] runtime_type = "io.containerd.kata.v2" privileged_without_host_devices = true - snapshotter = "overlayfs" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options] ConfigPath = "/usr/share/defaults/kata-containers/configuration.toml" -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview] - runtime_type = "io.containerd.kata.v2" - privileged_without_host_devices = true - snapshotter = "erofs" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview.options] - ConfigPath = "/usr/share/defaults/kata-containers/configuration-clh-templating.toml" [proxy_plugins] [proxy_plugins.tardev] type = "snapshot" @@ -2351,8 +2308,15 @@ health-check.localdns.local:53 { {{- end}} policy {{$forwardPolicy}} max_concurrent {{$override.MaxConcurrent}} + {{- if and $override.HealthCheck $override.HealthCheck.GetDuration}} + health_check {{$override.HealthCheck.GetDuration}}{{if $override.HealthCheck.GetNoRec}} no_rec{{end}}{{if $override.HealthCheck.GetDomain}} domain {{$override.HealthCheck.GetDomain}}{{end}} + {{- end}} + {{- if $override.GetFailfastAllUnhealthyUpstreams}} + failfast_all_unhealthy_upstreams + {{- end}} } ready {{$.NodeListenerIP}}:8181 + reload cache {{$override.CacheDurationInSeconds}} { success 9984 denial 9984 @@ -2416,8 +2380,15 @@ health-check.localdns.local:53 { {{- end}} policy {{$forwardPolicy}} max_concurrent {{$override.MaxConcurrent}} + {{- if and $override.HealthCheck $override.HealthCheck.GetDuration}} + health_check {{$override.HealthCheck.GetDuration}}{{if $override.HealthCheck.GetNoRec}} no_rec{{end}}{{if $override.HealthCheck.GetDomain}} domain {{$override.HealthCheck.GetDomain}}{{end}} + {{- end}} + {{- if $override.GetFailfastAllUnhealthyUpstreams}} + failfast_all_unhealthy_upstreams + {{- end}} } ready {{$.ClusterListenerIP}}:8181 + reload cache {{$override.CacheDurationInSeconds}} { success 9984 denial 9984 diff --git a/pkg/agent/baker_test.go b/pkg/agent/baker_test.go index 9ea69ecdc4e..67c4a2a7adc 100644 --- a/pkg/agent/baker_test.go +++ b/pkg/agent/baker_test.go @@ -474,6 +474,7 @@ health-check.localdns.local:53 { max_concurrent 1000 } ready 169.254.10.10:8181 + reload cache 3600 { success 9984 denial 9984 @@ -501,6 +502,7 @@ cluster.local:53 { max_concurrent 1000 } ready 169.254.10.10:8181 + reload cache 3600 { success 9984 denial 9984 @@ -518,6 +520,7 @@ testdomain456.com:53 { max_concurrent 1000 } ready 169.254.10.10:8181 + reload cache 3600 { success 9984 denial 9984 @@ -543,6 +546,7 @@ testdomain456.com:53 { max_concurrent 2000 } ready 169.254.10.11:8181 + reload cache 3600 { success 9984 denial 9984 @@ -663,6 +667,7 @@ health-check.localdns.local:53 { max_concurrent 1000 } ready 169.254.10.10:8181 + reload cache 3600 { success 9984 denial 9984 @@ -690,6 +695,7 @@ cluster.local:53 { max_concurrent 1000 } ready 169.254.10.10:8181 + reload cache 3600 { success 9984 denial 9984 @@ -707,6 +713,7 @@ testdomain456.com:53 { max_concurrent 1000 } ready 169.254.10.10:8181 + reload cache 3600 { success 9984 denial 9984 @@ -732,6 +739,7 @@ testdomain456.com:53 { max_concurrent 1000 } ready 169.254.10.11:8181 + reload cache 3600 { success 9984 denial 9984 @@ -759,6 +767,7 @@ cluster.local:53 { max_concurrent 1000 } ready 169.254.10.11:8181 + reload cache 3600 { success 9984 denial 9984 @@ -776,6 +785,7 @@ testdomain567.com:53 { max_concurrent 1000 } ready 169.254.10.11:8181 + reload cache 3600 { success 9984 denial 9984 diff --git a/pkg/agent/bakerapi.go b/pkg/agent/bakerapi.go index 29ba49e1916..9034e001005 100644 --- a/pkg/agent/bakerapi.go +++ b/pkg/agent/bakerapi.go @@ -40,7 +40,9 @@ func (agentBaker *agentBakerImpl) GetNodeBootstrapping(ctx context.Context, conf if config.AgentPoolProfile.IsWindows() { validateAndSetWindowsNodeBootstrappingConfiguration(config) } else { - ValidateAndSetLinuxNodeBootstrappingConfiguration(config) + if err := ValidateAndSetLinuxNodeBootstrappingConfigurationWithError(config); err != nil { + return nil, err + } } templateGenerator := InitializeTemplateGenerator() diff --git a/pkg/agent/bakerapi_test.go b/pkg/agent/bakerapi_test.go index 19ba47036a5..69b853285ff 100644 --- a/pkg/agent/bakerapi_test.go +++ b/pkg/agent/bakerapi_test.go @@ -179,6 +179,54 @@ var _ = Describe("AgentBaker API implementation tests", func() { Expect(nodeBootStrapping.SigImageConfig.Definition).To(Equal("2204gen2containerd")) }) + It("should accept supported custom Linux transparent huge page values", func() { + config.AgentPoolProfile.CustomLinuxOSConfig = &datamodel.CustomLinuxOSConfig{ + TransparentHugePageEnabled: "never", + TransparentHugePageDefrag: "defer+madvise", + } + agentBaker, err := NewAgentBaker() + Expect(err).NotTo(HaveOccurred()) + + _, err = agentBaker.GetNodeBootstrapping(context.Background(), config) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should reject unsupported custom Linux transparent huge page values", func() { + testCases := []struct { + name string + enabled string + defrag string + expectedErr string + }{ + { + name: "enabled", + enabled: "within_size", + expectedErr: "customLinuxOSConfig.transparentHugePageEnabled", + }, + { + name: "defrag", + defrag: "within_size", + expectedErr: "customLinuxOSConfig.transparentHugePageDefrag", + }, + } + agentBaker, err := NewAgentBaker() + Expect(err).NotTo(HaveOccurred()) + + for _, tc := range testCases { + configCopy, err := deepcopy.Anything(config) + Expect(err).To(BeNil()) + testConfig, ok := configCopy.(*datamodel.NodeBootstrappingConfiguration) + Expect(ok).To(BeTrue()) + testConfig.AgentPoolProfile.CustomLinuxOSConfig = &datamodel.CustomLinuxOSConfig{ + TransparentHugePageEnabled: tc.enabled, + TransparentHugePageDefrag: tc.defrag, + } + + _, err = agentBaker.GetNodeBootstrapping(context.Background(), testConfig) + Expect(err).To(MatchError(ContainSubstring(tc.expectedErr)), tc.name) + } + }) + It("should return the correct bootstrapping data when linux node image version override is present", func() { nodeImageVersionOverride := "202402.27.0" toggles := &testToggles{ diff --git a/pkg/agent/const.go b/pkg/agent/const.go index ad5da5d2584..24511398d11 100644 --- a/pkg/agent/const.go +++ b/pkg/agent/const.go @@ -30,7 +30,7 @@ const ( // kubernetesWindowsAgentCSECommandPS1 privides the command of Windows CSE. kubernetesWindowsAgentCSECommandPS1 = "windows/csecmd.ps1" // kubernetesWindowsAgentCustomDataPS1 is used for generating the customdata of Windows VM. - kubernetesWindowsAgentCustomDataPS1 = "windows/kuberneteswindowssetup.ps1" + kubernetesWindowsAgentCustomDataPS1 = "windows/kuberneteswindowssetup.ps1.template" /* Windows CSE helper scripts. These should all be listed in baker.go:func GetKubernetesWindowsAgentFunctions. */ kubernetesWindowsCSEHelperPS1 = "windows/windowscsehelper.ps1" @@ -72,6 +72,7 @@ const ( bindMountScript = "linux/cloud-init/artifacts/bind-mount.sh" bindMountSystemdService = "linux/cloud-init/artifacts/bind-mount.service" snapshotUpdateScript = "linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh" + securityUpdateScript = "linux/cloud-init/artifacts/ubuntu/security-update.sh" snapshotUpdateSystemdService = "linux/cloud-init/artifacts/ubuntu/snapshot-update.service" snapshotUpdateSystemdTimer = "linux/cloud-init/artifacts/ubuntu/snapshot-update.timer" packageUpdateScriptMariner = "linux/cloud-init/artifacts/mariner/mariner-package-update.sh" diff --git a/pkg/agent/datamodel/mocks.go b/pkg/agent/datamodel/mocks.go index 96f27ae4184..eacb54c8673 100644 --- a/pkg/agent/datamodel/mocks.go +++ b/pkg/agent/datamodel/mocks.go @@ -129,7 +129,7 @@ var ( ContainerdDownloadURLBase: "https://storage.googleapis.com/cri-containerd-release/", CSIProxyDownloadURL: "https://acs-mirror.azureedge.net/csi-proxy/v0.1.0/binaries/csi-proxy.tar.gz", WindowsProvisioningScriptsPackageURL: "https://acs-mirror.azureedge.net/aks-engine/windows/provisioning/signedscripts-v0.2.2.zip", - WindowsPauseImageURL: "mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.1", + WindowsPauseImageURL: "mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.2", AlwaysPullWindowsPauseImage: false, CseScriptsPackageURL: "https://acs-mirror.azureedge.net/aks/windows/cse/csescripts-v0.0.1.zip", CNIARM64PluginsDownloadURL: "https://acs-mirror.azureedge.net/cni-plugins/v0.8.7/binaries/cni-plugins-linux-arm64-v0.8.7.tgz", diff --git a/pkg/agent/datamodel/types.go b/pkg/agent/datamodel/types.go index 69f013e864b..2f0b774ee65 100644 --- a/pkg/agent/datamodel/types.go +++ b/pkg/agent/datamodel/types.go @@ -1696,7 +1696,7 @@ func FormatProdFQDNByLocation(fqdnPrefix string, location string, cloudSpecConfi type K8sComponents struct { // Full path to the "pause" image. Used for --pod-infra-container-image. - // For example: "mcr.microsoft.com/oss/v2/kubernetes/pause:3.6". + // For example: "mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.2". PodInfraContainerImageURL string // Full path to the hyperkube image. @@ -2596,18 +2596,49 @@ type LocalDNSCoreFileData struct { IncludeHostsPlugin bool } +// LocalDNSHealthCheck represents CoreDNS forward plugin health check settings. +type LocalDNSHealthCheck struct { + Duration *string `json:"duration,omitempty"` + NoRec *bool `json:"noRec,omitempty"` + Domain *string `json:"domain,omitempty"` +} + +func (h *LocalDNSHealthCheck) GetDuration() string { + if h != nil && h.Duration != nil { + return *h.Duration + } + return "" +} + +func (h *LocalDNSHealthCheck) GetNoRec() bool { + return h != nil && h.NoRec != nil && *h.NoRec +} + +func (h *LocalDNSHealthCheck) GetDomain() string { + if h != nil && h.Domain != nil { + return *h.Domain + } + return "" +} + // LocalDNSOverrides represents DNS override settings for both VnetDNS and KubeDNS traffic. // VnetDNS overrides apply to DNS traffic from pods with dnsPolicy:default or kubelet (referred to as VnetDNS traffic). // KubeDNS overrides apply to DNS traffic from pods with dnsPolicy:ClusterFirst (referred to as KubeDNS traffic). type LocalDNSOverrides struct { - QueryLogging string `json:"queryLogging,omitempty"` - Protocol string `json:"protocol,omitempty"` - ForwardDestination string `json:"forwardDestination,omitempty"` - ForwardPolicy string `json:"forwardPolicy,omitempty"` - MaxConcurrent *int32 `json:"maxConcurrent,omitempty"` - CacheDurationInSeconds *int32 `json:"cacheDurationInSeconds,omitempty"` - ServeStaleDurationInSeconds *int32 `json:"serveStaleDurationInSeconds,omitempty"` - ServeStale string `json:"serveStale,omitempty"` + QueryLogging string `json:"queryLogging,omitempty"` + Protocol string `json:"protocol,omitempty"` + ForwardDestination string `json:"forwardDestination,omitempty"` + ForwardPolicy string `json:"forwardPolicy,omitempty"` + MaxConcurrent *int32 `json:"maxConcurrent,omitempty"` + CacheDurationInSeconds *int32 `json:"cacheDurationInSeconds,omitempty"` + ServeStaleDurationInSeconds *int32 `json:"serveStaleDurationInSeconds,omitempty"` + ServeStale string `json:"serveStale,omitempty"` + FailfastAllUnhealthyUpstreams *bool `json:"failfastAllUnhealthyUpstreams,omitempty"` + HealthCheck *LocalDNSHealthCheck `json:"healthCheck,omitempty"` +} + +func (o *LocalDNSOverrides) GetFailfastAllUnhealthyUpstreams() bool { + return o != nil && o.FailfastAllUnhealthyUpstreams != nil && *o.FailfastAllUnhealthyUpstreams } // ShouldEnableLocalDNS returns true if AgentPoolProfile, LocalDNSProfile is not nil and diff --git a/pkg/agent/utils_test.go b/pkg/agent/utils_test.go index f274c30e0dc..5e4e7db5762 100644 --- a/pkg/agent/utils_test.go +++ b/pkg/agent/utils_test.go @@ -1476,7 +1476,9 @@ func TestValidateAndSetNodeBootstrappingConfiguration_StreamingConnectionIdleTim if tc.isWindows { validateAndSetWindowsNodeBootstrappingConfiguration(config) } else { - ValidateAndSetLinuxNodeBootstrappingConfiguration(config) + if err := ValidateAndSetLinuxNodeBootstrappingConfigurationWithError(config); err != nil { + t.Fatalf("unexpected validation error: %v", err) + } } _, exists := config.KubeletConfig["--streaming-connection-idle-timeout"] @@ -1506,7 +1508,9 @@ func TestValidateAndSetNodeBootstrappingConfiguration_StreamingConnectionIdleTim }, } - ValidateAndSetLinuxNodeBootstrappingConfiguration(config) + if err := ValidateAndSetLinuxNodeBootstrappingConfigurationWithError(config); err != nil { + t.Fatalf("unexpected validation error: %v", err) + } _, exists := config.KubeletConfig["--streaming-connection-idle-timeout"] if exists { @@ -1559,7 +1563,9 @@ func TestValidateAndSetNodeBootstrappingConfiguration_StreamingConnectionIdleTim }, } - ValidateAndSetLinuxNodeBootstrappingConfiguration(config) + if err := ValidateAndSetLinuxNodeBootstrappingConfigurationWithError(config); err != nil { + t.Fatalf("unexpected validation error: %v", err) + } cmdLine := GetOrderedKubeletConfigFlagString(config) if strings.Contains(cmdLine, "streaming-connection-idle-timeout") { @@ -1597,7 +1603,9 @@ func TestValidateAndSetNodeBootstrappingConfiguration_StreamingConnectionIdleTim }, } - ValidateAndSetLinuxNodeBootstrappingConfiguration(config) + if err := ValidateAndSetLinuxNodeBootstrappingConfigurationWithError(config); err != nil { + t.Fatalf("unexpected validation error: %v", err) + } cmdLine := GetOrderedKubeletConfigFlagString(config) if strings.Contains(cmdLine, "streaming-connection-idle-timeout") { @@ -1605,3 +1613,61 @@ func TestValidateAndSetNodeBootstrappingConfiguration_StreamingConnectionIdleTim } }) } + +func TestValidateAndSetLinuxNodeBootstrappingConfiguration_TransparentHugePageValues(t *testing.T) { + testCases := []struct { + name string + enabled string + defrag string + expectedErr string + }{ + { + name: "accepts empty values", + }, + { + name: "accepts supported enabled values", + enabled: "always", + }, + { + name: "accepts supported defrag values", + defrag: "defer+madvise", + }, + { + name: "rejects unsupported enabled values", + enabled: "within_size", + expectedErr: "customLinuxOSConfig.transparentHugePageEnabled", + }, + { + name: "rejects unsupported defrag values", + defrag: "within_size", + expectedErr: "customLinuxOSConfig.transparentHugePageDefrag", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + config := &datamodel.NodeBootstrappingConfiguration{ + AgentPoolProfile: &datamodel.AgentPoolProfile{ + CustomLinuxOSConfig: &datamodel.CustomLinuxOSConfig{ + TransparentHugePageEnabled: tc.enabled, + TransparentHugePageDefrag: tc.defrag, + }, + }, + } + + err := ValidateAndSetLinuxNodeBootstrappingConfigurationWithError(config) + if tc.expectedErr == "" { + if err != nil { + t.Fatalf("unexpected validation error: %v", err) + } + return + } + if err == nil { + t.Fatalf("expected validation error containing %q", tc.expectedErr) + } + if !strings.Contains(err.Error(), tc.expectedErr) { + t.Fatalf("expected validation error containing %q, got %q", tc.expectedErr, err.Error()) + } + }) + } +} diff --git a/pkg/agent/variables.go b/pkg/agent/variables.go index 545861b8198..54377c39e68 100644 --- a/pkg/agent/variables.go +++ b/pkg/agent/variables.go @@ -47,6 +47,7 @@ func getCustomDataVariables(config *datamodel.NodeBootstrappingConfiguration) pa "migPartitionScript": getBase64EncodedGzippedCustomScript(migPartitionScript, config), "ensureIMDSRestrictionScript": getBase64EncodedGzippedCustomScript(ensureIMDSRestrictionScript, config), "snapshotUpdateScript": getBase64EncodedGzippedCustomScript(snapshotUpdateScript, config), + "securityUpdateScript": getBase64EncodedGzippedCustomScript(securityUpdateScript, config), "snapshotUpdateService": getBase64EncodedGzippedCustomScript(snapshotUpdateSystemdService, config), "snapshotUpdateTimer": getBase64EncodedGzippedCustomScript(snapshotUpdateSystemdTimer, config), "packageUpdateScriptMariner": getBase64EncodedGzippedCustomScript(packageUpdateScriptMariner, config), diff --git a/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh index 48f4d8f8597..2912504f1da 100755 --- a/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh @@ -14,10 +14,231 @@ check_hosts_file_permissions() { stat -c '%a' "$AKS_LOCALDNS_HOSTS_FILE" } +check_test_fstab_permissions() { + printf "0%s" "$(getFileMode "$TEST_FSTAB_FILE")" +} + Describe 'cse_config.sh' Include "./parts/linux/cloud-init/artifacts/cse_config.sh" Include "./parts/linux/cloud-init/artifacts/cse_helpers.sh" + Describe 'configureTransparentHugePageSystemdService' + setup_thp_service() { + THP_ENABLED="never" + THP_DEFRAG="madvise" + unset FAIL_MKDIR FAIL_TEE_PATH FAIL_CHMOD FAIL_DAEMON_RELOAD FAIL_SYSTEMCTL_ENABLE + } + + cleanup_thp_service() { + unset THP_ENABLED THP_DEFRAG FAIL_MKDIR FAIL_TEE_PATH CAPTURE_TEE_PATH FAIL_CHMOD FAIL_DAEMON_RELOAD FAIL_SYSTEMCTL_ENABLE + } + + mkdir() { + echo "mkdir $*" + [ "${FAIL_MKDIR:-}" = "true" ] && return 1 + return 0 + } + + tee() { + if [ "${1:-}" = "${FAIL_TEE_PATH:-__none__}" ]; then + return 1 + fi + if [ "${1:-}" = "${CAPTURE_TEE_PATH:-__none__}" ]; then + cat >&2 + return 0 + fi + cat > /dev/null + } + + chmod() { + echo "chmod $*" + [ "${FAIL_CHMOD:-}" = "true" ] && return 1 + return 0 + } + + systemctl() { + echo "systemctl $*" + [ "${1:-}" = "daemon-reload" ] && [ "${FAIL_DAEMON_RELOAD:-}" = "true" ] && return 1 + return 0 + } + + systemctlEnableAndStart() { + echo "systemctlEnableAndStart $*" + [ "${FAIL_SYSTEMCTL_ENABLE:-}" = "true" ] && return 1 + return 0 + } + + BeforeEach 'setup_thp_service' + AfterEach 'cleanup_thp_service' + + It 'writes helper files, reloads systemd, and enables the service' + When run configureTransparentHugePageSystemdService + + The status should be success + The output should include "mkdir -p /opt/azure/containers /opt/azure/containers/aks-transparent-hugepage" + The output should include "chmod 0755 /opt/azure/containers/aks-transparent-hugepage.sh" + The output should include "systemctl daemon-reload" + The output should include "systemctlEnableAndStart aks-transparent-hugepage 30" + End + + It 'exits when helper directory creation fails' + FAIL_MKDIR="true" + + When run configureTransparentHugePageSystemdService + + The status should equal "$ERR_SYSCTL_RELOAD" + The output should include "mkdir -p /opt/azure/containers" + The output should not include "chmod" + The output should not include "systemctl daemon-reload" + End + + It 'exits when helper script write fails' + FAIL_TEE_PATH="/opt/azure/containers/aks-transparent-hugepage.sh" + + When run configureTransparentHugePageSystemdService + + The status should equal "$ERR_SYSCTL_RELOAD" + The output should include "mkdir -p /opt/azure/containers" + The output should not include "chmod" + The output should not include "systemctl daemon-reload" + End + + It 'exits when helper script chmod fails' + FAIL_CHMOD="true" + + When run configureTransparentHugePageSystemdService + + The status should equal "$ERR_SYSCTL_RELOAD" + The output should include "chmod 0755 /opt/azure/containers/aks-transparent-hugepage.sh" + The output should not include "systemctl daemon-reload" + End + + It 'exits when service unit write fails' + FAIL_TEE_PATH="/etc/systemd/system/aks-transparent-hugepage.service" + + When run configureTransparentHugePageSystemdService + + The status should equal "$ERR_SYSCTL_RELOAD" + The output should include "chmod 0755 /opt/azure/containers/aks-transparent-hugepage.sh" + The output should not include "systemctl daemon-reload" + End + + It 'exits when systemd daemon reload fails' + FAIL_DAEMON_RELOAD="true" + + When run configureTransparentHugePageSystemdService + + The status should equal "$ERR_SYSTEMCTL_START_FAIL" + The output should include "systemctl daemon-reload" + The output should not include "systemctlEnableAndStart" + End + + It 'does not embed raw THP values in the generated helper script' + THP_ENABLED='never"; touch /tmp/aks-thp-injection #' + CAPTURE_TEE_PATH="/opt/azure/containers/aks-transparent-hugepage.sh" + + When run configureTransparentHugePageSystemdService + + The status should be success + The output should include "systemctlEnableAndStart aks-transparent-hugepage 30" + The error should include 'cat "${thp_enabled_config}" > /sys/kernel/mm/transparent_hugepage/enabled' + The error should not include "touch /tmp/aks-thp-injection" + End + End + + Describe 'swapFileIsActive' + swapon() { + if [ "$*" != "--show --noheadings" ]; then + return 1 + fi + printf '%b' "${SWAPON_OUTPUT}" + } + + It 'matches an active swap file when swapon output has leading whitespace' + SWAPON_OUTPUT=' /swapfile\n' + + When call swapFileIsActive "/swapfile" + The status should be success + End + + It 'matches only the exact swap file path' + SWAPON_OUTPUT=' /swapfile-extra\n' + + When call swapFileIsActive "/swapfile" + The status should be failure + End + End + + Describe 'ensureSwapFileFstabEntry' + setup() { + TEST_FSTAB_DIR="$(mktemp -d)" + TEST_FSTAB_FILE="${TEST_FSTAB_DIR}/fstab" + : > "${TEST_FSTAB_FILE}" + } + + cleanup() { + rm -rf "${TEST_FSTAB_DIR}" + unset TEST_FSTAB_FILE + unset TEST_FSTAB_DIR + unset FAIL_MV + } + + BeforeEach 'setup' + AfterEach 'cleanup' + + mv() { + if [ "${FAIL_MV:-false}" = "true" ]; then + return 1 + fi + + command mv "$@" + } + + It 'replaces existing fstab entries for the same swap file' + chmod 0644 "${TEST_FSTAB_FILE}" + printf '/swapfile none swap sw 0 0\n/other none swap sw 0 0\n/swapfile none swap defaults 0 0\n' > "${TEST_FSTAB_FILE}" + expected_fstab='/other none swap sw 0 0 +/swapfile none swap noauto,nofail 0 0' + + When call ensureSwapFileFstabEntry "/swapfile" "${TEST_FSTAB_FILE}" + + The status should be success + The contents of file "${TEST_FSTAB_FILE}" should equal "${expected_fstab}" + End + + It 'preserves the fstab file mode when replacing entries' + chmod 0640 "${TEST_FSTAB_FILE}" + printf '/other none swap sw 0 0\n' > "${TEST_FSTAB_FILE}" + + When call ensureSwapFileFstabEntry "/swapfile" "${TEST_FSTAB_FILE}" + + The status should be success + The path "${TEST_FSTAB_FILE}" should be file + The result of function check_test_fstab_permissions should equal "0640" + End + + It 'keeps one canonical fstab entry when it already exists' + printf '/other none swap sw 0 0\n/swapfile none swap noauto,nofail 0 0\n' > "${TEST_FSTAB_FILE}" + expected_fstab='/other none swap sw 0 0 +/swapfile none swap noauto,nofail 0 0' + + When call ensureSwapFileFstabEntry "/swapfile" "${TEST_FSTAB_FILE}" + + The status should be success + The contents of file "${TEST_FSTAB_FILE}" should equal "${expected_fstab}" + End + + It 'leaves the existing fstab untouched when atomic replace fails' + printf '/other none swap sw 0 0\n' > "${TEST_FSTAB_FILE}" + FAIL_MV=true + + When call ensureSwapFileFstabEntry "/swapfile" "${TEST_FSTAB_FILE}" + + The status should be failure + The contents of file "${TEST_FSTAB_FILE}" should equal '/other none swap sw 0 0' + End + End + Describe 'logGPUDriverPrebakeReadiness' It 'reports marker_present=false when no prebake marker exists' GPU_DKMS_MARKER_FILE="$(mktemp)"; rm -f "${GPU_DKMS_MARKER_FILE}" @@ -1119,6 +1340,7 @@ PubkeyAuthentication no" API_SERVER_NAME="" AKS_CUSTOM_CLOUD_CONTAINER_REGISTRY_DNS_SUFFIX="" BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER="" + MCR_REPOSITORY_BASE="" } cleanup() { rm -rf "$TMP_DIR" @@ -1177,7 +1399,7 @@ providers: - /etc/kubernetes/azure.json - --registry-mirror=mcr.microsoft.com:test.azurecr.io' When call writeCredentialProviderConfig "$TMP_DIR/credential-provider-config.yaml" - The output should include "configure credential provider for network isolated cluster" + The output should include "configure credential provider with default settings" The contents of file "$TMP_DIR/credential-provider-config.yaml" should equal "$expected_config" End @@ -1206,6 +1428,34 @@ providers: The contents of file "$TMP_DIR/credential-provider-config.yaml" should equal "$expected_config" End + It 'should configure credential provider for custom cloud network isolated cluster' + AKS_CUSTOM_CLOUD_CONTAINER_REGISTRY_DNS_SUFFIX=".custom.registry.io" + BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER="test.azurecr.io" + expected_config='apiVersion: kubelet.config.k8s.io/v1 +kind: CredentialProviderConfig +providers: + - name: acr-credential-provider + matchImages: + - "*.azurecr.io" + - "*.azurecr.cn" + - "*.azurecr.de" + - "*.azurecr.us" + - "*.*.geo.azurecr.io" + - "*.*.geo.azurecr.cn" + - "*.*.geo.azurecr.de" + - "*.*.geo.azurecr.us" + - "*.custom.registry.io" + - "mcr.microsoft.com" + defaultCacheDuration: "10m" + apiVersion: credentialprovider.kubelet.k8s.io/v1 + args: + - /etc/kubernetes/azure.json + - --registry-mirror=mcr.microsoft.com:test.azurecr.io' + When call writeCredentialProviderConfig "$TMP_DIR/credential-provider-config.yaml" + The output should include "configure credential provider for custom cloud" + The contents of file "$TMP_DIR/credential-provider-config.yaml" should equal "$expected_config" + End + It 'should configure credential provider with identity binding enabled and all args' SERVICE_ACCOUNT_IMAGE_PULL_ENABLED="true" IDENTITY_BINDINGS_LOCAL_AUTHORITY_SNI="test.sni.local" @@ -1391,7 +1641,7 @@ providers: - --ib-default-tenant-id=my-tenant-id - --ib-apiserver-ip=apiserver.example.com' When call writeCredentialProviderConfig "$TMP_DIR/credential-provider-config.yaml" - The output should include "configure credential provider for network isolated cluster" + The output should include "configure credential provider with default settings" The contents of file "$TMP_DIR/credential-provider-config.yaml" should equal "$expected_config" End @@ -2537,36 +2787,36 @@ OVERRIDE_EOF ERR_PULL_POD_INFRA_CONTAINER_IMAGE=1 It 'should use MCR_REPOSITORY_BASE for image replacement when set' - get_sandbox_image() { echo "mcr.microsoft.us/oss/v2/kubernetes/pause:3.10.1"; } + get_sandbox_image() { echo "mcr.microsoft.us/oss/v2/kubernetes/pause:3.10.2"; } MCR_REPOSITORY_BASE="mcr.microsoft.us" BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER="myacr.azurecr.io/aks-managed-repository" When call ensurePodInfraContainerImage The status should be success - The output should include "Pulling with authentication for myacr.azurecr.io/aks-managed-repository/oss/v2/kubernetes/pause:3.10.1" + The output should include "Pulling with authentication for myacr.azurecr.io/aks-managed-repository/oss/v2/kubernetes/pause:3.10.2" End It 'should fall back to mcr.microsoft.com when MCR_REPOSITORY_BASE is unset' - get_sandbox_image() { echo "mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.1"; } + get_sandbox_image() { echo "mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.2"; } MCR_REPOSITORY_BASE="" BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER="myacr.azurecr.io/aks-managed-repository" When call ensurePodInfraContainerImage The status should be success - The output should include "Pulling with authentication for myacr.azurecr.io/aks-managed-repository/oss/v2/kubernetes/pause:3.10.1" + The output should include "Pulling with authentication for myacr.azurecr.io/aks-managed-repository/oss/v2/kubernetes/pause:3.10.2" End It 'should handle MCR_REPOSITORY_BASE with trailing slash' - get_sandbox_image() { echo "mcr.microsoft.us/oss/v2/kubernetes/pause:3.10.1"; } + get_sandbox_image() { echo "mcr.microsoft.us/oss/v2/kubernetes/pause:3.10.2"; } MCR_REPOSITORY_BASE="mcr.microsoft.us/" BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER="myacr.azurecr.io/aks-managed-repository" When call ensurePodInfraContainerImage The status should be success - The output should include "Pulling with authentication for myacr.azurecr.io/aks-managed-repository/oss/v2/kubernetes/pause:3.10.1" + The output should include "Pulling with authentication for myacr.azurecr.io/aks-managed-repository/oss/v2/kubernetes/pause:3.10.2" End End @@ -2683,4 +2933,144 @@ EOF The output should not include 'nvidia-device-plugin' End End + + Describe 'configureSwapFile' + SWAP_FILE_SIZE_MB=1 + + function [ { + if test "$1" = "-L" && test "$2" = "/dev/disk/azure/resource-part1"; then + return 0 + fi + local last_arg="" + for last_arg in "$@"; do :; done + if test "${last_arg}" = "]"; then + command [ "$@" + else + command [ "$@" ] + fi + } + + readlink() { + case "$2" in + /dev/disk/azure/resource-part1) echo "/dev/sdb1" ;; + /dev/disk/azure/root) echo "/dev/sda1" ;; + esac + } + + retrycmd_if_failure() { + echo "retrycmd_if_failure $*" + } + + chmod() { + echo "chmod $*" + } + + swapFileIsActive() { + echo "swapFileIsActive $1" + } + + reconcileSwapFilePersistence() { + echo "reconcileSwapFilePersistence $1" + } + + It 'falls back to OS disk when resource disk mountpoint cannot be determined' + findmnt() { + return 1 + } + df() { + printf 'Filesystem 1024-blocks Used Available Capacity Mounted on\n/dev/sda1 1000000 0 1000000 0%% /\n' + } + + When call configureSwapFile + + The status should be success + The output should include "Could not determine resource disk mountpoint, attempting to fall back to OS disk..." + The output should include "Will use OS disk for swap file" + The output should include "Swap file will be saved to: /swapfile" + The output should include "reconcileSwapFilePersistence /swapfile" + End + + It 'falls back to OS disk when resource disk free space cannot be determined' + findmnt() { + echo "/mnt/resource" + } + df() { + case "$2" in + /mnt/resource) + printf 'Filesystem 1024-blocks Used Available Capacity Mounted on\n' + ;; + /) + printf 'Filesystem 1024-blocks Used Available Capacity Mounted on\n/dev/sda1 1000000 0 1000000 0%% /\n' + ;; + esac + } + + When call configureSwapFile + + The status should be success + The output should include "Could not determine free space on resource disk, attempting to fall back to OS disk..." + The output should include "Will use OS disk for swap file" + The output should include "Swap file will be saved to: /swapfile" + The output should include "reconcileSwapFilePersistence /swapfile" + End + End + + Describe 'reconcileSwapFilePersistence' + findExistingSwapFileLocation() { + return 1 + } + + configureSwapFile() { + echo "createSwapFile" + } + + ensureSwapFileFstabEntry() { + echo "ensureSwapFileFstabEntry $1" + } + + configureSwapFileSystemdService() { + echo "configureSwapFileSystemdService $1" + } + + It 'creates the requested swap file when no existing swap file is present' + When call reconcileSwapFilePersistence + + The status should be success + The output should include "No existing AKS swap file found; creating swap file for persistence reconciliation" + The output should include "createSwapFile" + The output should not include "ensureSwapFileFstabEntry" + The output should not include "configureSwapFileSystemdService" + End + + It 'reconciles persistence for an existing swap file without recreating it' + When call reconcileSwapFilePersistence "/swapfile" + + The status should be success + The output should include "ensureSwapFileFstabEntry /swapfile" + The output should include "configureSwapFileSystemdService /swapfile" + The output should not include "createSwapFile" + End + + It 'exits when fstab reconciliation fails' + ensureSwapFileFstabEntry() { + return 1 + } + + When run reconcileSwapFilePersistence "/swapfile" + + The status should equal "$ERR_SWAP_CREATE_FAIL" + The output should not include "configureSwapFileSystemdService" + End + + It 'exits when swap systemd service reconciliation fails' + configureSwapFileSystemdService() { + return 1 + } + + When run reconcileSwapFilePersistence "/swapfile" + + The status should equal "$ERR_SWAP_CREATE_FAIL" + The output should include "ensureSwapFileFstabEntry /swapfile" + End + End End diff --git a/spec/parts/linux/cloud-init/artifacts/cse_helpers_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_helpers_spec.sh index efb439c7db2..84a86d3c04e 100644 --- a/spec/parts/linux/cloud-init/artifacts/cse_helpers_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_helpers_spec.sh @@ -939,3 +939,51 @@ EOF End End End + +Describe 'GPU driver image reference resolution' + # Both references are computed when cse_helpers.sh is sourced, so each case re-sources the + # script in a clean child shell with the environment the RP would supply. + resolve_driver_refs() { + env -u NVIDIA_DRIVER_IMAGE -u NVIDIA_DRIVER_IMAGE_PULL_REF -u MCR_REPOSITORY_BASE "$@" bash -c \ + 'source ./parts/linux/cloud-init/artifacts/cse_helpers.sh >/dev/null 2>&1; echo "$NVIDIA_DRIVER_IMAGE $NVIDIA_DRIVER_IMAGE_PULL_REF"' + } + + It 'uses public MCR for both references when MCR_REPOSITORY_BASE is unset' + When call resolve_driver_refs GPU_DRIVER_TYPE=grid + The status should be success + The output should eq "mcr.microsoft.com/aks/aks-gpu-grid mcr.microsoft.com/aks/aks-gpu-grid" + End + + It 'uses public MCR for both references when MCR_REPOSITORY_BASE is empty' + When call resolve_driver_refs MCR_REPOSITORY_BASE= GPU_DRIVER_TYPE=grid + The status should be success + The output should eq "mcr.microsoft.com/aks/aks-gpu-grid mcr.microsoft.com/aks/aks-gpu-grid" + End + + It 'pulls from the sovereign MCR endpoint and strips the trailing slash' + When call resolve_driver_refs MCR_REPOSITORY_BASE=mcr.microsoft.scloud/ GPU_DRIVER_TYPE=grid + The status should be success + The output should eq "mcr.microsoft.com/aks/aks-gpu-grid mcr.microsoft.scloud/aks/aks-gpu-grid" + End + + It 'pulls from the sovereign MCR endpoint supplied without a trailing slash' + When call resolve_driver_refs MCR_REPOSITORY_BASE=mcr.microsoft.eaglex.ic.gov GPU_DRIVER_TYPE=grid + The status should be success + The output should eq "mcr.microsoft.com/aks/aks-gpu-grid mcr.microsoft.eaglex.ic.gov/aks/aks-gpu-grid" + End + + It 'applies the cloud-aware base to every driver type' + When call resolve_driver_refs MCR_REPOSITORY_BASE=mcr.azure.cn/ GPU_DRIVER_TYPE=cuda-lts + The status should be success + The output should eq "mcr.microsoft.com/aks/aks-gpu-cuda-lts mcr.azure.cn/aks/aks-gpu-cuda-lts" + End + + # Regression guard: the VHD bakes aks-gpu-cuda-lts under its mcr.microsoft.com name and + # configGPUDrivers matches it exactly, so a cloud-specific value here would break every + # sovereign CUDA node's cache hit. + It 'keeps the cache-facing reference on public MCR regardless of cloud' + When call resolve_driver_refs MCR_REPOSITORY_BASE=mcr.microsoft.scloud/ GPU_DRIVER_TYPE=cuda-lts + The status should be success + The output should start with "mcr.microsoft.com/aks/aks-gpu-cuda-lts " + End +End diff --git a/spec/parts/linux/cloud-init/artifacts/cse_install_acl_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_install_acl_spec.sh index b63dbba6acf..76770aeb04c 100644 --- a/spec/parts/linux/cloud-init/artifacts/cse_install_acl_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_install_acl_spec.sh @@ -44,6 +44,49 @@ Describe 'cse_install_acl.sh' Include "./parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh" Include "./parts/linux/cloud-init/artifacts/cse_helpers.sh" + Describe 'installKubeletKubectlFromPkg' + It 'masks the kubelet sysext Upholds drop-in before activating the sysext' + MASK_CREATED=false + ln() { + if [ "$1" = "-sfn" ] && [ "$2" = "/dev/null" ] && [ "$3" = "/etc/systemd/system/multi-user.target.d/10-kubelet-kubelet.conf" ]; then + MASK_CREATED=true + fi + echo "mock ln $*" >&2 + } + mergeSysexts() { + if [ "$MASK_CREATED" != "true" ]; then + echo "mergeSysexts called before the kubelet sysext Upholds drop-in was masked" >&2 + return 1 + fi + echo "mock mergeSysexts $*" >&2 + } + When call installKubeletKubectlFromPkg "1.33" + The error should include "mock mkdir -p /etc/systemd/system/multi-user.target.d" + The error should include "mock ln -sfn /dev/null /etc/systemd/system/multi-user.target.d/10-kubelet-kubelet.conf" + The error should include "mock mergeSysexts kubelet mcr.microsoft.com/oss/v2/kubernetes/kubelet-sysext 1.33 kubectl mcr.microsoft.com/oss/v2/kubernetes/kubectl-sysext 1.33" + The error should include "mock ln -snf /usr/bin/kubelet /usr/bin/kubectl /opt/bin/" + The error should not include "mergeSysexts called before" + The status should be success + End + + It 'fails installation when the sysext Upholds drop-in cannot be masked' + mkdir() { + return 1 + } + mergeSysexts() { + echo "unexpected mergeSysexts" >&2 + } + installKubeletKubectlFromURL() { + echo "unexpected installKubeletKubectlFromURL" >&2 + } + When run installKubeletKubectlFromPkg "1.33" + The error should include "Failed to create kubelet sysext systemd drop-in directory /etc/systemd/system/multi-user.target.d" + The error should not include "unexpected mergeSysexts" + The error should not include "unexpected installKubeletKubectlFromURL" + The status should equal "$ERR_K8S_INSTALL_ERR" + End + End + Describe 'installSecureTLSBootstrapClientSysext' It 'calls mergeSysexts with correct URL and creates symlink on success' mergeSysexts() { diff --git a/spec/parts/linux/cloud-init/artifacts/cse_main_disable_modules_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_main_disable_modules_spec.sh index 9329541661e..6eddd14665a 100644 --- a/spec/parts/linux/cloud-init/artifacts/cse_main_disable_modules_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_main_disable_modules_spec.sh @@ -461,4 +461,15 @@ Describe 'CVE kernel module mitigation phase coverage' When call phase_body "nodePrep" The output should include "reconcileVulnerableKernelModuleMitigation" End + + It 'configures transparent huge page from basePrep so VHD bakes keep the intended state' + When call phase_body "basePrep" + The output should include "configureTransparentHugePage" + End + + It 'applies and reconciles transparent huge page from nodePrep for PIS and already-released VHDs' + When call phase_body "nodePrep" + The output should include "applyTransparentHugePageValues" + The output should include "reconcileTransparentHugePagePersistence" + End End diff --git a/spec/parts/linux/cloud-init/artifacts/localdns_spec.sh b/spec/parts/linux/cloud-init/artifacts/localdns_spec.sh index 65fbaae2cb9..6b061521fe2 100644 --- a/spec/parts/linux/cloud-init/artifacts/localdns_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/localdns_spec.sh @@ -23,6 +23,7 @@ Describe 'localdns.sh' TEST_DIR="/tmp/localdnstest" LOCALDNS_SCRIPT_PATH="${TEST_DIR}/opt/azure/containers/localdns" LOCALDNS_CORE_FILE="${LOCALDNS_SCRIPT_PATH}/localdns.corefile" + LIVEPATCHED_LOCALDNS_CORE_FILE="${LOCALDNS_SCRIPT_PATH}/livepatched.localdns.corefile" UPDATED_LOCALDNS_CORE_FILE="${LOCALDNS_SCRIPT_PATH}/updated.localdns.corefile" mkdir -p "$LOCALDNS_SCRIPT_PATH" # Use production-realistic corefile format with brace syntax @@ -391,6 +392,34 @@ EOF The stdout should include "Successfully exported forward IPs to ${LOCALDNS_SCRIPT_PATH}/forward_ips.prom" End + It 'should fetch LocalDNS LPS config through aks-node-controller when binary exists' + AKS_NODE_CONTROLLER_BINARY="${TEST_DIR}/aks-node-controller" + cat > "${AKS_NODE_CONTROLLER_BINARY}" <<'EOF' +#!/bin/bash +echo "anc args: $*" +exit 0 +EOF + chmod +x "${AKS_NODE_CONTROLLER_BINARY}" + + When run refresh_localdns_corefile_from_lps + The status should be success + The output should include "anc args: fetch-localdns-config --output ${LIVEPATCHED_LOCALDNS_CORE_FILE}" + The output should include "Completed LocalDNS LPS config fetch." + End + + It 'should skip LocalDNS LPS config fetch when aks-node-controller binary is missing' + AKS_NODE_CONTROLLER_BINARY="${TEST_DIR}/missing-aks-node-controller" + When run refresh_localdns_corefile_from_lps + The status should be success + The output should include "AKS node controller binary not found at ${AKS_NODE_CONTROLLER_BINARY}; skipping LocalDNS LPS config fetch." + End + + It 'should skip LocalDNS live patching status annotation when version file is missing' + When run annotate_node_with_localdns_livepatch_status + The status should be success + The output should include "LocalDNS corefile version file not found at ${LIVEPATCHED_LOCALDNS_CORE_FILE}.version, skipping live patching status annotation." + End + It 'should set correct permissions on forward_ips.prom file' When run replace_azurednsip_in_corefile The status should be success @@ -1960,3 +1989,50 @@ KUBECTL_EOF End End End + + + Describe 'livepatched corefile source selection' + setup() { + Include "./parts/linux/cloud-init/artifacts/localdns.sh" + TEST_DIR="/tmp/localdns-livepatched-test" + LOCALDNS_SCRIPT_PATH="${TEST_DIR}/opt/azure/containers/localdns" + LOCALDNS_CORE_FILE="${LOCALDNS_SCRIPT_PATH}/localdns.corefile" + LIVEPATCHED_LOCALDNS_CORE_FILE="${LOCALDNS_SCRIPT_PATH}/livepatched.localdns.corefile" + UPDATED_LOCALDNS_CORE_FILE="${LOCALDNS_SCRIPT_PATH}/updated.localdns.corefile" + RESOLV_CONF="${TEST_DIR}/run/systemd/resolve/resolv.conf" + mkdir -p "${LOCALDNS_SCRIPT_PATH}" "$(dirname "${RESOLV_CONF}")" + echo 'nameserver 10.0.0.1' > "${RESOLV_CONF}" + } + cleanup() { + rm -rf "${TEST_DIR}" + } + BeforeEach 'setup' + AfterEach 'cleanup' + + It 'uses the livepatched Corefile when present' + printf '.:53 { + forward . 9.9.9.9 +} +' > "${LOCALDNS_CORE_FILE}" + printf '.:53 { + forward . 168.63.129.16 +} +' > "${LIVEPATCHED_LOCALDNS_CORE_FILE}" + When run replace_azurednsip_in_corefile + The status should be success + The output should include 'Successfully updated' + The contents of file "${UPDATED_LOCALDNS_CORE_FILE}" should include '10.0.0.1' + The contents of file "${UPDATED_LOCALDNS_CORE_FILE}" should not include '9.9.9.9' + End + + It 'falls back to the generated Corefile when no livepatched Corefile exists' + printf '.:53 { + forward . 168.63.129.16 +} +' > "${LOCALDNS_CORE_FILE}" + When run replace_azurednsip_in_corefile + The status should be success + The output should include 'Successfully updated' + The contents of file "${UPDATED_LOCALDNS_CORE_FILE}" should include '10.0.0.1' + End + End diff --git a/spec/parts/linux/cloud-init/artifacts/security-update_spec.sh b/spec/parts/linux/cloud-init/artifacts/security-update_spec.sh new file mode 100644 index 00000000000..014f4f24337 --- /dev/null +++ b/spec/parts/linux/cloud-init/artifacts/security-update_spec.sh @@ -0,0 +1,206 @@ +#!/bin/bash +# shellcheck disable=SC2317 + +Describe 'security-update.sh' + security_patch_test_node_json() { + local repo_service="${1:-}" + + jq -nc --arg repoService "${repo_service}" \ + '{metadata: {name: "aks-node-1", labels: {"kubernetes.azure.com/agentpool": "ap1"}, annotations: {"kubernetes.azure.com/live-patching-repo-service": $repoService}}}' + } + + setup() { + Include ./parts/linux/cloud-init/artifacts/ubuntu/security-update.sh + TEST_DIR="/tmp/security-update-test" + rm -rf "${TEST_DIR}" + mkdir -p "${TEST_DIR}" + + SECURITY_PATCH_CONFIG_DIR="${TEST_DIR}/security-patch" + TEST_NODE_JSON="$(security_patch_test_node_json)" + TEST_APT_UPDATE_STATUS=0 + TEST_ANNOTATE_STATUS=0 + KUBECTL="kubectl" + export SECURITY_PATCH_CONFIG_DIR TEST_NODE_JSON KUBECTL + export TEST_APT_UPDATE_STATUS TEST_ANNOTATE_STATUS + } + + cleanup() { + rm -rf "${TEST_DIR}" + } + + BeforeEach 'setup' + AfterEach 'cleanup' + + Mock lsb_release + echo "jammy" + End + + Mock unattended-upgrade + echo "unattended-upgrade called" + End + + Mock sleep + echo "sleep called" + End + + Mock kubectl + echo "kubectl called with args: $*" + exit "${TEST_ANNOTATE_STATUS}" + End + + apt_get_update_with_opts() { + echo "apt_get_update_with_opts called with args: $*" + return "${TEST_APT_UPDATE_STATUS}" + } + + It 'applies the timestamp selected for the node agent pool' + When call updateSecurityPatch '{"agentPools":{"ap1":{"goldenTimestamp":"20260710T000000Z"},"ap2":{"goldenTimestamp":"20260701T000000Z"}}}' "${TEST_NODE_JSON}" + The status should be success + The output should include 'apt_get_update_with_opts called with args: -o Acquire::http::Timeout=300 -o Acquire::https::Timeout=300 -o Acquire::Retries=3' + The output should include 'unattended-upgrade called' + The output should include 'kubectl called with args: annotate --overwrite node aks-node-1 kubernetes.azure.com/live-patching-current-timestamp=20260710T000000Z' + The output should include 'securityPatch update completed successfully: 20260710T000000Z' + The contents of file "${SECURITY_PATCH_CONFIG_DIR}/sources.list" should include 'deb https://snapshot.ubuntu.com/ubuntu/20260710T000000Z jammy main restricted' + The contents of file "${SECURITY_PATCH_CONFIG_DIR}/apt.conf" should include "Dir::Etc::sourcelist \"${SECURITY_PATCH_CONFIG_DIR}/sources.list\";" + End + + It 'compares only the current node agent pool timestamp' + desired_payload='{"agentPools":{"ap1":{"goldenTimestamp":"20260710T000000Z"},"ap2":{"goldenTimestamp":"20260715T000000Z"}}}' + current_payload='{"agentPools":{"ap1":{"goldenTimestamp":"20260710T000000Z"},"ap2":{"goldenTimestamp":"20260701T000000Z"}}}' + + When call securityPatchIsCurrent "${desired_payload}" "${current_payload}" "${TEST_NODE_JSON}" + The status should be success + End + + It 'detects a changed timestamp for the current node agent pool' + desired_payload='{"agentPools":{"ap1":{"goldenTimestamp":"20260710T000000Z"}}}' + current_payload='{"agentPools":{"ap1":{"goldenTimestamp":"20260701T000000Z"}}}' + + When call securityPatchIsCurrent "${desired_payload}" "${current_payload}" "${TEST_NODE_JSON}" + The status should be failure + End + + It 'retries unattended upgrade until it succeeds' + eval 'unattended-upgrade() { + TEST_UNATTENDED_ATTEMPT=$((TEST_UNATTENDED_ATTEMPT + 1)) + echo "unattended-upgrade called: ${TEST_UNATTENDED_ATTEMPT}" + [ "${TEST_UNATTENDED_ATTEMPT}" -ge 3 ] + }' + TEST_UNATTENDED_ATTEMPT=0 + export TEST_UNATTENDED_ATTEMPT + + When call security_patch_unattended_upgrade + The status should be success + The output should include 'unattended-upgrade called: 3' + The output should include 'executed unattended upgrade 3 times' + End + + It 'fails after ten unattended upgrade attempts' + eval 'unattended-upgrade() { + TEST_UNATTENDED_ATTEMPT=$((TEST_UNATTENDED_ATTEMPT + 1)) + echo "unattended-upgrade called: ${TEST_UNATTENDED_ATTEMPT}" + return 1 + }' + TEST_UNATTENDED_ATTEMPT=0 + export TEST_UNATTENDED_ATTEMPT + + When call security_patch_unattended_upgrade + The status should be failure + The output should include 'unattended-upgrade called: 10' + End + + It 'uses the private repository service for a network-isolated cluster' + TEST_NODE_JSON="$(security_patch_test_node_json '10.0.0.1')" + export TEST_NODE_JSON + + When call updateSecurityPatch '{"agentPools":{"ap1":{"goldenTimestamp":"20260710T000000Z"}}}' "${TEST_NODE_JSON}" + The status should be success + The output should include 'securityPatch update completed successfully: 20260710T000000Z' + The contents of file "${SECURITY_PATCH_CONFIG_DIR}/sources.list" should include 'deb http://10.0.0.1/ubuntu jammy main restricted' + The contents of file "${SECURITY_PATCH_CONFIG_DIR}/sources.list" should not include 'snapshot.ubuntu.com' + End + + It 'falls back to the snapshot service for an invalid repository annotation' + TEST_NODE_JSON="$(security_patch_test_node_json 'example.com')" + export TEST_NODE_JSON + + When call updateSecurityPatch '{"agentPools":{"ap1":{"goldenTimestamp":"20260710T000000Z"}}}' "${TEST_NODE_JSON}" + The status should be success + The output should include 'securityPatch update completed successfully: 20260710T000000Z' + The stderr should include 'ignoring invalid live patching repo service: example.com' + The contents of file "${SECURITY_PATCH_CONFIG_DIR}/sources.list" should include 'snapshot.ubuntu.com/ubuntu/20260710T000000Z' + End + + It 'fails when the component has no profile for the node agent pool' + When call updateSecurityPatch '{"agentPools":{"ap2":{"goldenTimestamp":"20260710T000000Z"}}}' "${TEST_NODE_JSON}" + The status should be failure + The output should include 'securityPatch profile is missing for agent pool: ap1' + The output should not include 'apt_get_update_with_opts called' + End + + It 'fails when the selected golden timestamp is malformed' + When call updateSecurityPatch '{"agentPools":{"ap1":{"goldenTimestamp":"not-a-timestamp"}}}' "${TEST_NODE_JSON}" + The status should be failure + The output should include 'securityPatch goldenTimestamp is invalid: not-a-timestamp' + The output should not include 'apt_get_update_with_opts called' + End + + It 'returns failure when apt metadata refresh fails' + TEST_APT_UPDATE_STATUS=1 + export TEST_APT_UPDATE_STATUS + + When call updateSecurityPatch '{"agentPools":{"ap1":{"goldenTimestamp":"20260710T000000Z"}}}' "${TEST_NODE_JSON}" + The status should be failure + The output should include 'apt_get_update_with_opts failed' + The output should not include 'unattended-upgrade called' + The output should not include 'kubectl called' + End + + It 'returns failure when the legacy status annotation cannot be updated' + TEST_ANNOTATE_STATUS=1 + export TEST_ANNOTATE_STATUS + + When call updateSecurityPatch '{"agentPools":{"ap1":{"goldenTimestamp":"20260710T000000Z"}}}' "${TEST_NODE_JSON}" + The status should be failure + The output should include 'unattended-upgrade called' + The output should include 'kubectl called with args: annotate --overwrite node aks-node-1 kubernetes.azure.com/live-patching-current-timestamp=20260710T000000Z' + The output should include 'failed to update legacy securityPatch status annotation' + The output should not include 'securityPatch update completed successfully' + End + + It 'returns failure before apt update when apt configuration cannot be written' + SECURITY_PATCH_CONFIG_DIR="/dev/null/security-patch" + export SECURITY_PATCH_CONFIG_DIR + + When call updateSecurityPatch '{"agentPools":{"ap1":{"goldenTimestamp":"20260710T000000Z"}}}' "${TEST_NODE_JSON}" + The status should be failure + The output should include 'failed to generate securityPatch apt configuration' + The stderr should include 'Not a directory' + The output should not include 'apt_get_update_with_opts called' + The output should not include 'unattended-upgrade called' + End + + It 'does not mask a sources list write failure when apt config is writable' + mkdir -p "${SECURITY_PATCH_CONFIG_DIR}/sources.list" + + When call updateSecurityPatch '{"agentPools":{"ap1":{"goldenTimestamp":"20260710T000000Z"}}}' "${TEST_NODE_JSON}" + The status should be failure + The output should include 'failed to generate securityPatch apt configuration' + The stderr should include 'Is a directory' + The path "${SECURITY_PATCH_CONFIG_DIR}/apt.conf" should not be exist + The output should not include 'apt_get_update_with_opts called' + The output should not include 'unattended-upgrade called' + End + + It 'returns failure before apt update when the Ubuntu codename is unavailable' + Mock lsb_release + exit 1 + End + + When call updateSecurityPatch '{"agentPools":{"ap1":{"goldenTimestamp":"20260710T000000Z"}}}' "${TEST_NODE_JSON}" + The status should be failure + The output should include 'failed to determine Ubuntu codename' + The output should not include 'apt_get_update_with_opts called' + The output should not include 'unattended-upgrade called' + End +End \ No newline at end of file diff --git a/spec/parts/linux/cloud-init/artifacts/snapshot-update-service_spec.sh b/spec/parts/linux/cloud-init/artifacts/snapshot-update-service_spec.sh new file mode 100644 index 00000000000..f67923cb31c --- /dev/null +++ b/spec/parts/linux/cloud-init/artifacts/snapshot-update-service_spec.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +Describe 'snapshot update service commands' + validate_ubuntu_packer_inputs() { + local packer_file + + for packer_file in \ + vhdbuilder/packer/vhd-image-builder-base.json \ + vhdbuilder/packer/vhd-image-builder-cvm.json \ + vhdbuilder/packer/vhd-image-builder-arm64-gen2.json \ + vhdbuilder/packer/vhd-image-builder-arm64-gb.json + do + jq -e ' + [.. | objects | .source? // empty] as $sources | + ($sources | index("parts/linux/cloud-init/artifacts/ubuntu/security-update.sh")) != null and + ($sources | index("parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh")) != null and + ($sources | index("parts/linux/cloud-init/artifacts/ubuntu/snapshot-update.service")) != null and + ($sources | index("parts/linux/cloud-init/artifacts/ubuntu/snapshot-update.timer")) != null and + ($sources | index("parts/linux/cloud-init/artifacts/ubuntu/knead-component.sh")) == null + ' "${packer_file}" > /dev/null || return 1 + done + } + + validate_mariner_packer_inputs() { + local packer_file + + for packer_file in \ + vhdbuilder/packer/vhd-image-builder-mariner.json \ + vhdbuilder/packer/vhd-image-builder-mariner-cvm.json \ + vhdbuilder/packer/vhd-image-builder-mariner-arm64.json + do + jq -e ' + [.. | objects | .source? // empty] as $sources | + ($sources | index("parts/linux/cloud-init/artifacts/mariner/mariner-package-update.sh")) != null and + ($sources | index("parts/linux/cloud-init/artifacts/mariner/package-update.service")) != null and + ($sources | index("parts/linux/cloud-init/artifacts/mariner/package-update.timer")) != null + ' "${packer_file}" > /dev/null || return 1 + done + } + + validate_ubuntu_hotfix_transition() { + local template="parts/linux/cloud-init/nodecustomdata.yml" + local generator="hotfix/hotfix_generate.py" + local key + + for key in snapshotUpdateScript securityUpdateScript + do + grep -Fq "GetVariableProperty \"cloudInitData\" \"${key}\"" "${template}" || return 1 + grep -Fq ": \"${key}\"" "${generator}" || return 1 + done + + # Existing VHD-baked units remain unchanged; hotfixes update scripts only. + ! grep -Fq 'GetVariableProperty "cloudInitData" "snapshotUpdateService"' "${template}" || return 1 + ! grep -Fq 'GetVariableProperty "cloudInitData" "snapshotUpdateTimer"' "${template}" + } + + It 'keeps the Ubuntu snapshot service unchanged' + When run grep -Fx 'ExecStart=/opt/azure/containers/ubuntu-snapshot-update.sh' parts/linux/cloud-init/artifacts/ubuntu/snapshot-update.service + The status should be success + The output should equal 'ExecStart=/opt/azure/containers/ubuntu-snapshot-update.sh' + End + + It 'keeps the Azure Linux package updater' + When run grep -Fx 'ExecStart=/opt/azure/containers/mariner-package-update.sh' parts/linux/cloud-init/artifacts/mariner/package-update.service + The status should be success + The output should equal 'ExecStart=/opt/azure/containers/mariner-package-update.sh' + End + + It 'stages the generic updater, handler, and existing units for Ubuntu images' + When call validate_ubuntu_packer_inputs + The status should be success + End + + It 'preserves Azure Linux package updater inputs' + When call validate_mariner_packer_inputs + The status should be success + End + + It 'hotfix-delivers the generic updater and security handler without switching units' + When call validate_ubuntu_hotfix_transition + The status should be success + End +End \ No newline at end of file diff --git a/spec/parts/linux/cloud-init/artifacts/ubuntu-snapshot-update_spec.sh b/spec/parts/linux/cloud-init/artifacts/ubuntu-snapshot-update_spec.sh index 8f5cedcc5e2..07e748a00f3 100644 --- a/spec/parts/linux/cloud-init/artifacts/ubuntu-snapshot-update_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/ubuntu-snapshot-update_spec.sh @@ -1,15 +1,32 @@ #!/bin/bash +# shellcheck disable=SC2089,SC2090 -Describe 'ubuntu-snapshot-update.sh' +Describe 'ubuntu-snapshot-update.sh generic reconciliation' setup() { Include ./parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh - TEST_DIR="/tmp/live-patching-test" - mkdir -p ${TEST_DIR} - SECURITY_PATCH_CONFIG_DIR="${TEST_DIR}" + TEST_DIR="/tmp/knead-component-test" + rm -rf "${TEST_DIR}" + mkdir -p "${TEST_DIR}" + KUBECONFIG="${TEST_DIR}/kubeconfig" touch "${KUBECONFIG}" KUBECTL="kubectl" + KNEAD_COMPONENT_STATE_FILE="${TEST_DIR}/state/current.json" + TEST_COMPONENTS_JSON_FILE="${TEST_DIR}/components.json" + TEST_KUBECTL_ARGS_FILE="${TEST_DIR}/kubectl-args" + + TEST_STATUS="" + TEST_GOAL="" + TEST_AGENT_POOL="ap1" + TEST_REPO_SERVICE="" + printf '%s' '{"components":[]}' > "${TEST_COMPONENTS_JSON_FILE}" + TEST_SECURITY_STATUS=0 + TEST_LOCALDNS_STATUS=0 + TEST_ANNOTATE_STATUS=0 + export KUBECTL KNEAD_COMPONENT_STATE_FILE TEST_COMPONENTS_JSON_FILE TEST_KUBECTL_ARGS_FILE + export TEST_STATUS TEST_GOAL TEST_AGENT_POOL TEST_REPO_SERVICE TEST_SECURITY_STATUS TEST_LOCALDNS_STATUS TEST_ANNOTATE_STATUS } + cleanup() { rm -rf "${TEST_DIR}" } @@ -17,108 +34,382 @@ Describe 'ubuntu-snapshot-update.sh' BeforeEach 'setup' AfterEach 'cleanup' - Mock apt_get_update_with_opts - echo "apt_get_update_with_opts mock called" + Mock hostname + echo "AKS-NODE-1" End - Mock unattended-upgrade - echo "unattended-upgrade mock called" + + Mock kubectl + case "$*" in + *"annotate --overwrite node"*) + echo "annotate mock called with args: $*" + if [ "${TEST_ANNOTATE_STATUS}" -ne 0 ]; then + exit "${TEST_ANNOTATE_STATUS}" + fi + ;; + *"get node"*) + jq -nc \ + --arg name "aks-node-1" \ + --arg goal "${TEST_GOAL}" \ + --arg status "${TEST_STATUS}" \ + --arg agentPool "${TEST_AGENT_POOL}" \ + --arg repoService "${TEST_REPO_SERVICE}" \ + '{metadata: {name: $name, labels: {"kubernetes.azure.com/agentpool": $agentPool}, annotations: {"kubernetes.azure.com/live-patching-config-goal-hash": $goal, "kubernetes.azure.com/live-patching-status": $status, "kubernetes.azure.com/live-patching-repo-service": $repoService}}}' + ;; + *"get cm"*) + printf '%s' "$*" > "${TEST_KUBECTL_ARGS_FILE}" + cat "${TEST_COMPONENTS_JSON_FILE}" + ;; + esac End - It 'should update successfully for regular cluster' - Mock lsb_release - echo "jammy" - End - sources_list=$(cat < "${TEST_COMPONENTS_JSON_FILE}" + TEST_GOAL="$(sha256sum "${TEST_COMPONENTS_JSON_FILE}" | awk '{print $1}')" + export TEST_GOAL + } + + fail_result_then_write_status() { + KNEAD_COMPONENT_RESULTS='not-json' + KNEAD_COMPONENT_RESULTS_VALID=true + knead_set_component_result securityPatch Succeeded || true + echo "results valid: ${KNEAD_COMPONENT_RESULTS_VALID}" + knead_write_status aks-node-1 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + } + + fail_component_parse_then_write_status() { + knead_apply_components 'not-json' '{}' || true + echo "results valid: ${KNEAD_COMPONENT_RESULTS_VALID}" + knead_write_status aks-node-1 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + } + + It 'does nothing when goal annotation is not set' + TEST_GOAL="" + export TEST_GOAL + + When call knead_main + The status should be success + The output should include 'live-patching goal is not set, skip knead-component' + The output should not include 'updateSecurityPatch called' + The output should not include 'annotate mock called' + End + + It 'fails after the kubeconfig wait deadline' + rm -f "${KUBECONFIG}" + KNEAD_KUBECONFIG_WAIT_TIMEOUT_SECONDS=0 + export KNEAD_KUBECONFIG_WAIT_TIMEOUT_SECONDS + + When call knead_main + The status should be failure + The error should include 'timed out waiting for kubelet kubeconfig after 0s' + The output should not include 'updateSecurityPatch called' + End + + It 'does nothing when status is converged for the goal' + TEST_GOAL="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + TEST_STATUS='{"currentHash":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","components":{"securityPatch":{"code":"Succeeded"}}}' + export TEST_GOAL TEST_STATUS + + When call knead_main + The status should be success + The output should include 'live-patching goal is already converged, nothing to apply' + The output should not include 'updateSecurityPatch called' + The output should not include 'annotate mock called' + End + + It 'dispatches securityPatch and writes successful status' + set_payload_goal '{"components":[{"name":"securityPatch","nodeConfig":"{\"agentPools\":{\"ap1\":{\"goldenTimestamp\":\"20260623T000000Z\"}}}"}]}' + + When call knead_main + The status should be success + The output should include 'applying component: securityPatch' + The output should include 'updateSecurityPatch called with args: {"agentPools":{"ap1":{"goldenTimestamp":"20260623T000000Z"}}}' + The output should include '"kubernetes.azure.com/agentpool":"ap1"' + The output should include 'annotate mock called with args: annotate --overwrite node aks-node-1 kubernetes.azure.com/live-patching-status={"currentHash":"' + The output should include '"components":{"securityPatch":{"code":"Succeeded"}}}' + The output should include 'knead-component completed successfully' + The contents of file "${KNEAD_COMPONENT_STATE_FILE}" should include '"securityPatch"' + End + + It 'reads the dotted ConfigMap key with an escaped JSONPath' + set_payload_goal '{"components":[]}' + + When call knead_read_configmap "${TEST_GOAL}" + The status should be success + The output should equal '{"components":[]}' + The contents of file "${TEST_KUBECTL_ARGS_FILE}" should equal 'get cm -n kube-system live-patching-config -o jsonpath={.data.live-patching-config\.json}' + End + + + It 'dispatches localDNS and writes successful status' + mkdir -p /opt/azure/containers + cat > /opt/azure/containers/aks-node-controller <<'EOF' +#!/bin/bash +echo "aks-node-controller called with args: $*" +cat > /tmp/localdns-livepatch-payload +echo applied EOF -) - Mock kubectl - if [[ "$@" == *"live-patching-golden-timestamp"* ]]; then - echo "20250815T000000Z" - fi + chmod +x /opt/azure/containers/aks-node-controller + Mock systemctl + echo "systemctl called with args: $*" End - When run main + set_payload_goal '{"components":[{"name":"localDNS","nodeConfig":"{\"profiles\":{\"ap1\":{\"configChecksum\":\"localdns-v1\"}}}"}]}' + + When call knead_main The status should be success - The output should include 'live patching repo service is not set, use ubuntu snapshot repo' - The output should include 'apt_get_update_with_opts mock called' - The output should include 'unattended-upgrade mock called' - The output should include 'Executed unattended upgrade 1 times' - The output should include 'snapshot update completed successfully' - The contents of file "${SECURITY_PATCH_CONFIG_DIR}/sources.list" should eq "${sources_list}" - The contents of file "${SECURITY_PATCH_CONFIG_DIR}/apt.conf" should eq "${apt_config}" - End - - It 'should update successfully for ni cluster' - Mock lsb_release - echo "noble" - End - sources_list=$(cat < /opt/azure/containers/aks-node-controller <<'EOF' +#!/bin/bash +echo applied EOF -) - Mock kubectl - if [[ "$@" == *"live-patching-golden-timestamp"* ]]; then - echo "20250825T000000Z" - elif [[ "$@" == *"live-patching-repo-service"* ]]; then - echo "10.0.0.1" - fi + chmod +x /opt/azure/containers/aks-node-controller + Mock systemctl + echo "systemctl called with args: $*" + exit 1 End - When run main + set_payload_goal '{"components":[{"name":"localDNS","nodeConfig":"{\"profiles\":{\"ap1\":{\"configChecksum\":\"localdns-v1\"}}}"}]}' + + When call knead_main + The status should be failure + The output should include 'applying component: localDNS' + The output should include 'failed to restart localdns.service' + The output should include 'component failed: localDNS' + The output should include '"components":{"localDNS":{"code":"Failed"}}}' + End + + It 'fails before dispatch when the goal hash does not match the ConfigMap payload' + printf '%s' '{"components":[{"name":"securityPatch","nodeConfig":"{\"agentPools\":{}}"}]}' > "${TEST_COMPONENTS_JSON_FILE}" + TEST_GOAL="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + export TEST_GOAL + + When call knead_main + The status should be failure + The error should include 'live-patching goal hash does not match ConfigMap payload' + The output should not include 'updateSecurityPatch called' + The output should not include 'annotate mock called' + End + + It 'rejects a prefixed goal hash' + printf '%s' '{"components":[]}' > "${TEST_COMPONENTS_JSON_FILE}" + TEST_GOAL="sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + export TEST_GOAL + + When call knead_main + The status should be failure + The error should include 'live-patching goal hash must be a 64-character lowercase sha256 digest' + The output should not include 'annotate mock called' + End + + It 'fails before dispatch when components is not an array' + set_payload_goal '{"components":{}}' + + When call knead_main + The status should be failure + The error should include 'live-patching-config payload has invalid envelope' + The output should not include 'updateSecurityPatch called' + The output should not include 'annotate mock called' + End + + It 'fails before dispatch when component names are duplicated' + set_payload_goal '{"components":[{"name":"securityPatch","nodeConfig":"{\"agentPools\":{}}"},{"name":"securityPatch","nodeConfig":"{\"agentPools\":{}}"}]}' + + When call knead_main + The status should be failure + The error should include 'live-patching-config payload has invalid envelope' + The output should not include 'updateSecurityPatch called' + The output should not include 'annotate mock called' + End + + It 'records a failed securityPatch result and advances currentHash' + set_payload_goal '{"components":[{"name":"securityPatch","nodeConfig":"{\"agentPools\":{\"ap1\":{\"goldenTimestamp\":\"20260623T000000Z\"}}}"}]}' + TEST_SECURITY_STATUS=1 + export TEST_SECURITY_STATUS + + When call knead_main + The status should be failure + The output should include 'updateSecurityPatch called' + The output should include 'component failed: securityPatch' + The output should include 'failed components: securityPatch' + The output should include 'annotate mock called with args: annotate --overwrite node aks-node-1 kubernetes.azure.com/live-patching-status={"currentHash":"' + The output should include '"securityPatch":{"code":"Failed"}' + End + + It 'continues reporting later components after securityPatch fails' + set_payload_goal '{"components":[{"name":"securityPatch","nodeConfig":"{\"agentPools\":{\"ap1\":{\"goldenTimestamp\":\"20260623T000000Z\"}}}"},{"name":"npd","nodeConfig":"{\"version\":\"v20260623.0\"}"}]}' + TEST_SECURITY_STATUS=1 + export TEST_SECURITY_STATUS + + When call knead_main + The status should be failure + The output should include 'component failed: securityPatch' + The output should include 'unsupported component: npd' + The output should include 'failed components: securityPatch' + The output should include '"securityPatch":{"code":"Failed"}' + The output should not include '"npd"' + End + + It 'retries when currentHash matches but a component previously failed' + set_payload_goal '{"components":[{"name":"securityPatch","nodeConfig":"{\"agentPools\":{\"ap1\":{\"goldenTimestamp\":\"20260623T000000Z\"}}}"}]}' + TEST_STATUS="{\"currentHash\":\"${TEST_GOAL}\",\"components\":{\"securityPatch\":{\"code\":\"Failed\"}}}" + export TEST_STATUS + + When call knead_main The status should be success - The output should include 'live patching repo service is: 10.0.0.1' - The output should include 'apt_get_update_with_opts mock called' - The output should include 'unattended-upgrade mock called' - The output should include 'Executed unattended upgrade 1 times' - The output should include 'snapshot update completed successfully' - The contents of file "${SECURITY_PATCH_CONFIG_DIR}/sources.list" should eq "${sources_list}" - The contents of file "${SECURITY_PATCH_CONFIG_DIR}/apt.conf" should eq "${apt_config}" - End - - It 'should do nothing if golden timestamp is not set' - Mock kubectl - echo "" - End - When run main + The output should include 'updateSecurityPatch called' + The output should include '"securityPatch":{"code":"Succeeded"}' + End + + It 'ignores an unsupported component without blocking securityPatch dispatch' + set_payload_goal '{"components":[{"name":"npd","nodeConfig":"{\"version\":\"v20260623.0\"}"},{"name":"securityPatch","nodeConfig":"{\"agentPools\":{\"ap1\":{\"goldenTimestamp\":\"20260623T000000Z\"}}}"}]}' + + When call knead_main The status should be success - The output should include 'golden timestamp is not set, skip live patching' + The output should include 'unsupported component: npd' + The output should include 'updateSecurityPatch called' + The output should not include 'failed components:' + The output should not include '"npd"' + The output should include '"securityPatch":{"code":"Succeeded"}' + The contents of file "${KNEAD_COMPONENT_STATE_FILE}" should include '"securityPatch"' + The contents of file "${KNEAD_COMPONENT_STATE_FILE}" should not include '"npd"' End - It 'should do nothing if golden timestamp equals current timestamp' - Mock kubectl - echo "20250820T000000Z" - End - When run main + It 'does not rerun a successful component when an unsupported sibling is ignored' + set_payload_goal '{"components":[{"name":"npd","nodeConfig":"{\"version\":\"v20260623.0\"}"},{"name":"securityPatch","nodeConfig":"{\"agentPools\":{\"ap1\":{\"goldenTimestamp\":\"20260623T000000Z\"}}}"}]}' + knead_main > /dev/null || true + + When call knead_main The status should be success - The output should include 'golden timestamp is: 20250820T000000Z' - The output should include 'current timestamp is: 20250820T000000Z' - The output should include 'golden and current timestamp is the same, nothing to patch' + The output should include 'unsupported component: npd' + The output should include 'component is already current: securityPatch' + The output should not include 'updateSecurityPatch called' + The output should not include '"npd"' + The output should include '"securityPatch":{"code":"Succeeded"}' + End + + It 'does not rerun an unchanged component when another component changes' + mkdir -p "$(dirname "${KNEAD_COMPONENT_STATE_FILE}")" + printf '%s' '{"components":[{"name":"securityPatch","nodeConfig":"{\"agentPools\":{\"ap1\":{\"goldenTimestamp\":\"20260623T000000Z\"}}}"}]}' > "${KNEAD_COMPONENT_STATE_FILE}" + set_payload_goal '{"components":[{"name":"npd","nodeConfig":"{\"version\":\"v20260701.0\"}"},{"name":"securityPatch","nodeConfig":"{\"agentPools\":{\"ap1\":{\"goldenTimestamp\":\"20260623T000000Z\"}}}"}]}' + + When call knead_main + The status should be success + The output should include 'unsupported component: npd' + The output should include 'component is already current: securityPatch' + The output should not include 'updateSecurityPatch called' + The output should not include '"npd"' + The output should include '"securityPatch":{"code":"Succeeded"}' + End + + It 'does not rerun securityPatch when only another agent pool changes' + mkdir -p "$(dirname "${KNEAD_COMPONENT_STATE_FILE}")" + printf '%s' '{"components":[{"name":"securityPatch","nodeConfig":"{\"agentPools\":{\"ap1\":{\"goldenTimestamp\":\"20260623T000000Z\"},\"ap2\":{\"goldenTimestamp\":\"20260623T000000Z\"}}}"}]}' > "${KNEAD_COMPONENT_STATE_FILE}" + set_payload_goal '{"components":[{"name":"securityPatch","nodeConfig":"{\"agentPools\":{\"ap1\":{\"goldenTimestamp\":\"20260623T000000Z\"},\"ap2\":{\"goldenTimestamp\":\"20260701T000000Z\"}}}"}]}' + + When call knead_main + The status should be success + The output should include 'component is already current: securityPatch' + The output should not include 'updateSecurityPatch called' + The output should include '"securityPatch":{"code":"Succeeded"}' + End + + It 'preserves sibling state when recording a successful component' + mkdir -p "$(dirname "${KNEAD_COMPONENT_STATE_FILE}")" + printf '%s' '{"components":[{"name":"npd","nodeConfig":"{\"version\":\"v20260623.0\"}"}]}' > "${KNEAD_COMPONENT_STATE_FILE}" + + When call knead_write_component_state securityPatch '{"agentPools":{"ap1":{"goldenTimestamp":"20260623T000000Z"}}}' + The status should be success + The contents of file "${KNEAD_COMPONENT_STATE_FILE}" should include '"npd"' + The contents of file "${KNEAD_COMPONENT_STATE_FILE}" should include 'v20260623.0' + The contents of file "${KNEAD_COMPONENT_STATE_FILE}" should include '"securityPatch"' + The contents of file "${KNEAD_COMPONENT_STATE_FILE}" should include '20260623T000000Z' + End + + It 'fails after handler success when status annotation update fails' + set_payload_goal '{"components":[{"name":"securityPatch","nodeConfig":"{\"agentPools\":{\"ap1\":{\"goldenTimestamp\":\"20260623T000000Z\"}}}"}]}' + TEST_ANNOTATE_STATUS=1 + export TEST_ANNOTATE_STATUS + + When call knead_main + The status should be failure + The output should include 'updateSecurityPatch called' + The output should include 'failed to update live-patching status annotation' + The output should not include 'knead-component completed successfully' + The contents of file "${KNEAD_COMPONENT_STATE_FILE}" should include '"securityPatch"' + End + + It 'fails before annotation when component results cannot be rendered' + KNEAD_COMPONENT_RESULTS='not-json' + export KNEAD_COMPONENT_RESULTS + + When call knead_write_status aks-node-1 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + The status should be failure + The output should include 'failed to render live-patching status annotation' + The stderr should include 'parse error' + The output should not include 'annotate mock called' + End + + It 'fails before annotation when status rendering produces no output' + KNEAD_COMPONENT_RESULTS='' + export KNEAD_COMPONENT_RESULTS + + When call knead_write_status aks-node-1 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + The status should be failure + The output should include 'failed to render live-patching status annotation' + The output should not include 'annotate mock called' + End + + It 'preserves existing results and refuses status after a result update fails' + When call fail_result_then_write_status + The status should be failure + The output should include 'failed to record component result: securityPatch=Succeeded' + The output should include 'results valid: false' + The output should include 'refusing to write incomplete live-patching status' + The stderr should include 'parse error' + The output should not include 'annotate mock called' + End + + It 'refuses status after component parsing fails' + When call fail_component_parse_then_write_status + The status should be failure + The output should include 'failed to read component count' + The output should include 'results valid: false' + The output should include 'refusing to write incomplete live-patching status' + The stderr should include 'parse error' + The output should not include 'annotate mock called' End End diff --git a/staging/cse/windows/README b/staging/cse/windows/README index 419c8c03986..82f932e5169 100644 --- a/staging/cse/windows/README +++ b/staging/cse/windows/README @@ -19,7 +19,7 @@ ```bash branchName="master" -currentCseVersion="v0.0.51" # `WindowsCSEScriptsPackage` defined in `parts/windows/kuberneteswindowssetup.ps1` +currentCseVersion="v0.0.51" # `WindowsCSEScriptsPackage` defined in `parts/windows/kuberneteswindowssetup.ps1.template` testCseVersion="v0.0.51.0" # Test package name. NOTE: Please do not use the official package format and earlier used version. url="https://raw.githubusercontent.com/Azure/AgentBaker/$branchName/staging/cse/windows" diff --git a/staging/cse/windows/all.ps1 b/staging/cse/windows/all.ps1 new file mode 100644 index 00000000000..4fc889eebc8 --- /dev/null +++ b/staging/cse/windows/all.ps1 @@ -0,0 +1,12 @@ + +# Dot-source cse scripts with functions that are bundled on the VHD +. c:\AzureData\windows\azurecnifunc.ps1 +. c:\AzureData\windows\calicofunc.ps1 +. c:\AzureData\windows\configfunc.ps1 +. c:\AzureData\windows\containerdfunc.ps1 +. c:\AzureData\windows\kubeletfunc.ps1 +. c:\AzureData\windows\kubernetesfunc.ps1 +. c:\AzureData\windows\nvidiagpudriverfunc.ps1 +. c:\AzureData\windows\securetlsbootstrapfunc.ps1 +. c:\AzureData\windows\windowsciliumnetworkingfunc.ps1 +. c:\AzureData\windows\networkisolatedclusterfunc.ps1 diff --git a/staging/cse/windows/containerdfunc.ps1 b/staging/cse/windows/containerdfunc.ps1 index 79f132ff4b0..579c0dfab50 100644 --- a/staging/cse/windows/containerdfunc.ps1 +++ b/staging/cse/windows/containerdfunc.ps1 @@ -27,6 +27,7 @@ function RegisterContainerDService { & "$KubeDir\nssm.exe" set containerd Start SERVICE_DEMAND_START | RemoveNulls & "$KubeDir\nssm.exe" set containerd ObjectName LocalSystem | RemoveNulls & "$KubeDir\nssm.exe" set containerd Type SERVICE_WIN32_OWN_PROCESS | RemoveNulls + & "$KubeDir\nssm.exe" set containerd AppPriority ABOVE_NORMAL_PRIORITY_CLASS | RemoveNulls & "$KubeDir\nssm.exe" set containerd AppThrottle 1500 | RemoveNulls & "$KubeDir\nssm.exe" set containerd AppStdout "$KubeDir\containerd.log" | RemoveNulls & "$KubeDir\nssm.exe" set containerd AppStderr "$KubeDir\containerd.err.log" | RemoveNulls diff --git a/staging/cse/windows/containerdfunc.tests.ps1 b/staging/cse/windows/containerdfunc.tests.ps1 index ff88cfc9997..c510d4f6df0 100644 --- a/staging/cse/windows/containerdfunc.tests.ps1 +++ b/staging/cse/windows/containerdfunc.tests.ps1 @@ -85,7 +85,7 @@ Describe "Containerd Functions Tests" { $containerdDir = "$PSScriptRoot\containerdfunc.tests.suites" $cniBinDir = 'C:/cni/bin' $cniConfDir = 'C:/cni/conf' - $pauseImage = 'mcr.microsoft.com/oss/v2/kubernetes/pause:3.6' + $pauseImage = 'mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.2' $global:KubeClusterConfigPath = [Io.path]::Combine("", "kubeclusterconfig.json") $global:ContainerdInstallLocation = $containerdDir @@ -146,7 +146,7 @@ Describe "Containerd Functions Tests" { $containerdDir = "$PSScriptRoot\containerdfunc.tests.suites" $cniBinDir = 'C:/cni/bin' $cniConfDir = 'C:/cni/conf' - $pauseImage = 'mcr.microsoft.com/oss/v2/kubernetes/pause:3.6' + $pauseImage = 'mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.2' $global:KubeClusterConfigPath = [Io.path]::Combine("", "kubeclusterconfig.json") $global:ContainerdInstallLocation = $containerdDir diff --git a/staging/cse/windows/kubernetesfunc.ps1 b/staging/cse/windows/kubernetesfunc.ps1 index 9a613380d30..50d1126cb8a 100644 --- a/staging/cse/windows/kubernetesfunc.ps1 +++ b/staging/cse/windows/kubernetesfunc.ps1 @@ -115,7 +115,7 @@ function Write-KubeClusterConfig { $Global:ClusterConfiguration | Add-Member -MemberType NoteProperty -Name Cri -Value @{ Name = "containerd"; Images = @{ - # e.g. "mcr.microsoft.com/oss/v2/kubernetes/pause:3.6" + # e.g. "mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.2" "Pause" = $global:WindowsPauseImageURL } } diff --git a/staging/cse/windows/networkisolatedclusterfunc.tests.ps1 b/staging/cse/windows/networkisolatedclusterfunc.tests.ps1 index 6c3129ab67a..2c7613bae6d 100644 --- a/staging/cse/windows/networkisolatedclusterfunc.tests.ps1 +++ b/staging/cse/windows/networkisolatedclusterfunc.tests.ps1 @@ -143,7 +143,7 @@ Describe "Set-PodInfraContainerImage" { { "Cri": { "Images": { - "Pause": "mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.1" + "Pause": "mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.2" } } } @@ -176,7 +176,7 @@ Describe "Set-PodInfraContainerImage" { It "returns early when image already exists locally" { $script:CtrExeMock = { param($Args) - return @("mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.1") + return @("mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.2") } function global:Mock-OrasCli { @@ -241,7 +241,7 @@ Describe "Set-PodInfraContainerImage" { $global:MCRRepositoryBase = $null { Set-PodInfraContainerImage } | Should -Not -Throw - $script:orasImageArg | Should -Be "myacr.azurecr.io/aks-managed-repository/oss/v2/kubernetes/pause:3.10.1" + $script:orasImageArg | Should -Be "myacr.azurecr.io/aks-managed-repository/oss/v2/kubernetes/pause:3.10.2" } It "should use MCRRepositoryBase (and trim trailing slash) for image replacement" { @@ -250,7 +250,7 @@ Describe "Set-PodInfraContainerImage" { { "Cri": { "Images": { - "Pause": "mcr.microsoft.us/oss/v2/kubernetes/pause:3.10.1" + "Pause": "mcr.microsoft.us/oss/v2/kubernetes/pause:3.10.2" } } } @@ -268,7 +268,7 @@ Describe "Set-PodInfraContainerImage" { $global:MCRRepositoryBase = "mcr.microsoft.us/" { Set-PodInfraContainerImage } | Should -Not -Throw - $script:orasImageArg | Should -Be "myacr.azurecr.io/aks-managed-repository/oss/v2/kubernetes/pause:3.10.1" + $script:orasImageArg | Should -Be "myacr.azurecr.io/aks-managed-repository/oss/v2/kubernetes/pause:3.10.2" } } diff --git a/vhdbuilder/packer/packer_source.sh b/vhdbuilder/packer/packer_source.sh index 182c6332702..64f014a199b 100644 --- a/vhdbuilder/packer/packer_source.sh +++ b/vhdbuilder/packer/packer_source.sh @@ -216,6 +216,8 @@ copyPackerFiles() { KUBELET_SERVICE_DEST=/etc/systemd/system/kubelet.service SECURE_TLS_BOOTSTRAP_SERVICE_SRC=/home/packer/secure-tls-bootstrap.service SECURE_TLS_BOOTSTRAP_SERVICE_DEST=/etc/systemd/system/secure-tls-bootstrap.service + SECURITY_UPDATE_SH_SRC=/home/packer/security-update.sh + SECURITY_UPDATE_SH_DEST=/opt/azure/containers/security-update.sh USU_SH_SRC=/home/packer/ubuntu-snapshot-update.sh USU_SH_DEST=/opt/azure/containers/ubuntu-snapshot-update.sh MPU_SH_SRC=/home/packer/mariner-package-update.sh @@ -615,6 +617,7 @@ copyPackerFiles() { cpAndMode $PAM_D_COMMON_ACCOUNT_SRC $PAM_D_COMMON_ACCOUNT_DEST 644 cpAndMode $PAM_D_COMMON_AUTH_SRC $PAM_D_COMMON_AUTH_DEST 644 cpAndMode $PAM_D_COMMON_PASSWORD_SRC $PAM_D_COMMON_PASSWORD_DEST 644 + cpAndMode $SECURITY_UPDATE_SH_SRC $SECURITY_UPDATE_SH_DEST 544 cpAndMode $USU_SH_SRC $USU_SH_DEST 544 if [ "$UBUNTU_RELEASE" = "24.04" ] && [ "$CPU_ARCH" = "arm64" ]; then diff --git a/vhdbuilder/packer/test/linux-vhd-content-test.sh b/vhdbuilder/packer/test/linux-vhd-content-test.sh index f662230adb7..8e6b1d1d36a 100644 --- a/vhdbuilder/packer/test/linux-vhd-content-test.sh +++ b/vhdbuilder/packer/test/linux-vhd-content-test.sh @@ -2460,6 +2460,37 @@ checkLocaldnsScriptsAndConfigs() { #------------------------ End of test code related to localdns ------------------------ +testKneadSecurityPatchingAssets() { + local test="testKneadSecurityPatchingAssets" + local os_sku="$1" + local file + local permissions + local -A expected_files=( + ["/etc/systemd/system/snapshot-update.service"]=644 + ["/etc/systemd/system/snapshot-update.timer"]=644 + ["/opt/azure/containers/security-update.sh"]=544 + ["/opt/azure/containers/ubuntu-snapshot-update.sh"]=544 + ) + + if [ "$os_sku" != "Ubuntu" ]; then + return 0 + fi + + for file in "${!expected_files[@]}"; do + if [ ! -f "$file" ]; then + err "$test" "Expected file not found: $file" + fi + permissions=$(stat -c "%a" "$file") + if [ "$permissions" != "${expected_files[$file]}" ]; then + err "$test" "Incorrect permissions for $file. Expected ${expected_files[$file]}, got $permissions" + fi + done + + if ! grep -Fxq 'ExecStart=/opt/azure/containers/ubuntu-snapshot-update.sh' /etc/systemd/system/snapshot-update.service; then + err "$test" "snapshot-update.service does not execute the generic reconciler" + fi +} + # Basic sanity check for Inspektor Gadget artifacts baked into the image. testInspektorGadgetAssets() { local test="testInspektorGadgetAssets" @@ -2711,6 +2742,7 @@ testLtsKernel $OS_VERSION $OS_SKU $ENABLE_FIPS testAutologinDisabled $OS_SKU testCorednsBinaryExtractedAndCached $OS_VERSION checkLocaldnsScriptsAndConfigs $OS_SKU +testKneadSecurityPatchingAssets $OS_SKU testInspektorGadgetAssets testPackageDownloadURLFallbackLogic testFileOwnership $OS_SKU diff --git a/vhdbuilder/packer/vhd-image-builder-arm64-gb.json b/vhdbuilder/packer/vhd-image-builder-arm64-gb.json index 667be2946ae..80c9f5ae6ac 100644 --- a/vhdbuilder/packer/vhd-image-builder-arm64-gb.json +++ b/vhdbuilder/packer/vhd-image-builder-arm64-gb.json @@ -332,6 +332,11 @@ "source": "parts/linux/cloud-init/artifacts/setup-custom-search-domains.sh", "destination": "/home/packer/setup-custom-search-domains.sh" }, + { + "type": "file", + "source": "parts/linux/cloud-init/artifacts/ubuntu/security-update.sh", + "destination": "/home/packer/security-update.sh" + }, { "type": "file", "source": "parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh", diff --git a/vhdbuilder/packer/vhd-image-builder-arm64-gen2.json b/vhdbuilder/packer/vhd-image-builder-arm64-gen2.json index 8cc63a7241c..309b51d8369 100644 --- a/vhdbuilder/packer/vhd-image-builder-arm64-gen2.json +++ b/vhdbuilder/packer/vhd-image-builder-arm64-gen2.json @@ -317,6 +317,11 @@ "source": "parts/linux/cloud-init/artifacts/setup-custom-search-domains.sh", "destination": "/home/packer/setup-custom-search-domains.sh" }, + { + "type": "file", + "source": "parts/linux/cloud-init/artifacts/ubuntu/security-update.sh", + "destination": "/home/packer/security-update.sh" + }, { "type": "file", "source": "parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh", diff --git a/vhdbuilder/packer/vhd-image-builder-base.json b/vhdbuilder/packer/vhd-image-builder-base.json index a35a0807a18..09dcacccdea 100644 --- a/vhdbuilder/packer/vhd-image-builder-base.json +++ b/vhdbuilder/packer/vhd-image-builder-base.json @@ -320,6 +320,11 @@ "source": "parts/linux/cloud-init/artifacts/setup-custom-search-domains.sh", "destination": "/home/packer/setup-custom-search-domains.sh" }, + { + "type": "file", + "source": "parts/linux/cloud-init/artifacts/ubuntu/security-update.sh", + "destination": "/home/packer/security-update.sh" + }, { "type": "file", "source": "parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh", diff --git a/vhdbuilder/packer/vhd-image-builder-cvm.json b/vhdbuilder/packer/vhd-image-builder-cvm.json index f13b5ff46a0..d18ac41adb9 100644 --- a/vhdbuilder/packer/vhd-image-builder-cvm.json +++ b/vhdbuilder/packer/vhd-image-builder-cvm.json @@ -324,6 +324,11 @@ "source": "parts/linux/cloud-init/artifacts/setup-custom-search-domains.sh", "destination": "/home/packer/setup-custom-search-domains.sh" }, + { + "type": "file", + "source": "parts/linux/cloud-init/artifacts/ubuntu/security-update.sh", + "destination": "/home/packer/security-update.sh" + }, { "type": "file", "source": "parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh",