From 4558b88c09e4fc2e9b2af01a61563cd7a6a70d17 Mon Sep 17 00:00:00 2001 From: Devin Wong Date: Thu, 3 Sep 2026 11:52:50 -0700 Subject: [PATCH 01/26] feat: embed rendered script hotfixes in ANC Generate distro-specific nodecustomdata YAML with AgentBaker's canonical renderer and apply it transactionally from the selected ANC package. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/hotfix-generate.yml | 112 +++-- aks-node-controller/README.md | 28 ++ aks-node-controller/app.go | 22 + aks-node-controller/app_test.go | 65 +++ aks-node-controller/scripthotfix/applier.go | 422 ++++++++++++++++++ .../scripthotfix/applier_test.go | 301 +++++++++++++ .../scripthotfix/generated/active | 1 + .../generated/rendered_nodecustomdata_acl.yml | 2 + .../rendered_nodecustomdata_azlosguard.yml | 2 + .../rendered_nodecustomdata_flatcar.yml | 2 + .../rendered_nodecustomdata_mariner.yml | 2 + .../rendered_nodecustomdata_ubuntu.yml | 2 + e2e/scenario_test.go | 67 +++ e2e/types.go | 13 + e2e/vmss.go | 137 +++++- e2e/vmss_test.go | 81 ++++ hotfix/hotfix_generate.py | 257 ++++++----- hotfix/hotfix_generate_test.py | 238 ++++++++++ hotfix/render-nodecustomdata/main.go | 98 ++++ pkg/agent/baker.go | 23 + pkg/agent/nodecustomdata_render_test.go | 80 ++++ 21 files changed, 1800 insertions(+), 155 deletions(-) create mode 100644 aks-node-controller/scripthotfix/applier.go create mode 100644 aks-node-controller/scripthotfix/applier_test.go create mode 100644 aks-node-controller/scripthotfix/generated/active create mode 100644 aks-node-controller/scripthotfix/generated/rendered_nodecustomdata_acl.yml create mode 100644 aks-node-controller/scripthotfix/generated/rendered_nodecustomdata_azlosguard.yml create mode 100644 aks-node-controller/scripthotfix/generated/rendered_nodecustomdata_flatcar.yml create mode 100644 aks-node-controller/scripthotfix/generated/rendered_nodecustomdata_mariner.yml create mode 100644 aks-node-controller/scripthotfix/generated/rendered_nodecustomdata_ubuntu.yml create mode 100644 hotfix/hotfix_generate_test.py create mode 100644 hotfix/render-nodecustomdata/main.go create mode 100644 pkg/agent/nodecustomdata_render_test.go diff --git a/.github/workflows/hotfix-generate.yml b/.github/workflows/hotfix-generate.yml index fed11c9548f..617d7e7aeac 100644 --- a/.github/workflows/hotfix-generate.yml +++ b/.github/workflows/hotfix-generate.yml @@ -2,9 +2,8 @@ name: Hotfix Template Update # Auto-detects whether a hotfix is needed for a PR targeting an official/* release # branch and, if so, computes the version numbers and updates the generated files: # - If aks-node-controller/ changed vs the base branch, bumps `version`. -# - If parts/linux/cloud-init/nodecustomdata.yml ends up changed vs the base branch -# (either directly, or via auto-injection of changed CSE scripts), bumps -# `scripts_version`. +# - Changed CSE scripts are rendered into platform-specific nodecustomdata YAML +# files and embedded in the hotfix ANC package. # - Writes the result to # parts/linux/cloud-init/artifacts/aks-node-controller-hotfix.json (embedded # directly into scriptless customData by pkg/agent/baker.go). @@ -83,57 +82,80 @@ jobs: - name: Generate hotfix files run: | + python3 -m unittest hotfix.hotfix_generate_test python3 hotfix/hotfix_generate.py "origin/${GITHUB_BASE_REF}" - name: Commit changes via API env: GH_TOKEN: ${{ steps.app-token.outputs.token }} - HEAD_REF: ${{ github.head_ref }} run: | - FILES=( - "parts/linux/cloud-init/nodecustomdata.yml" - "parts/linux/cloud-init/artifacts/aks-node-controller-hotfix.json" + FILES=() + while IFS= read -r STATUS_LINE; do + FILES+=("${STATUS_LINE:3}") + done < <( + git status --porcelain --untracked-files=all -- \ + parts/linux/cloud-init/artifacts/aks-node-controller-hotfix.json \ + aks-node-controller/scripthotfix/generated ) - CHANGED=0 + if [ "${#FILES[@]}" -eq 0 ]; then + echo "No template changes needed." + exit 0 + fi + + REPOSITORY="repos/${{ github.repository }}" + BRANCH="${GITHUB_HEAD_REF}" + CHECKOUT_SHA=$(git rev-parse HEAD) + START_SHA=$(gh api "${REPOSITORY}/git/ref/heads/${BRANCH}" --jq '.object.sha') + if [ "$START_SHA" != "$CHECKOUT_SHA" ]; then + echo "Branch moved after checkout; refusing to commit stale generated content." >&2 + exit 1 + fi + BASE_TREE=$(gh api "${REPOSITORY}/git/commits/${START_SHA}" --jq '.tree.sha') + TREE='[]' for FILE in "${FILES[@]}"; do - # git diff --quiet misses untracked files (e.g. the target hotfix json on - # its first-ever generation on this branch), so use `git status - # --porcelain` instead, which reports both modified and untracked paths. - if [ -z "$(git status --porcelain -- "$FILE")" ]; then - continue - fi - CHANGED=1 - # For a brand-new file the Contents API lookup 404s (no sha yet); the - # step runs under `bash -e`, so guard the lookup and only pass -f sha - # when the file already exists on the branch, otherwise the PUT must - # omit sha entirely to create the file. - SHA=$(gh api "repos/${{ github.repository }}/contents/${FILE}?ref=${HEAD_REF}" --jq '.sha' 2>/dev/null || true) - if [ ! -e "$FILE" ]; then - if [ -n "$SHA" ]; then - gh api "repos/${{ github.repository }}/contents/${FILE}" \ - -X DELETE \ - -f message="chore: auto-generate hotfix content for ${FILE}" \ - -f branch="${HEAD_REF}" \ - -f sha="$SHA" - fi - continue - fi - CONTENT=$(base64 -w 0 "$FILE") - if [ -n "$SHA" ]; then - gh api "repos/${{ github.repository }}/contents/${FILE}" \ - -X PUT \ - -f message="chore: auto-generate hotfix content for ${FILE}" \ - -f content="$CONTENT" \ - -f branch="${HEAD_REF}" \ - -f sha="$SHA" + if [ -e "$FILE" ]; then + BLOB_SHA=$( + base64 -w 0 "$FILE" | + jq -Rs '{content: ., encoding: "base64"}' | + gh api "${REPOSITORY}/git/blobs" -X POST --input - --jq '.sha' + ) + TREE=$( + jq -c \ + --arg path "$FILE" \ + --arg sha "$BLOB_SHA" \ + '. + [{path: $path, mode: "100644", type: "blob", sha: $sha}]' \ + <<< "$TREE" + ) else - gh api "repos/${{ github.repository }}/contents/${FILE}" \ - -X PUT \ - -f message="chore: auto-generate hotfix content for ${FILE}" \ - -f content="$CONTENT" \ - -f branch="${HEAD_REF}" + TREE=$( + jq -c \ + --arg path "$FILE" \ + '. + [{path: $path, mode: "100644", type: "blob", sha: null}]' \ + <<< "$TREE" + ) fi done - if [ "$CHANGED" -eq 0 ]; then - echo "No template changes needed." + NEW_TREE=$( + jq -n \ + --arg base_tree "$BASE_TREE" \ + --argjson tree "$TREE" \ + '{base_tree: $base_tree, tree: $tree}' | + gh api "${REPOSITORY}/git/trees" -X POST --input - --jq '.sha' + ) + NEW_COMMIT=$( + jq -n \ + --arg message "chore: auto-generate hotfix content" \ + --arg tree "$NEW_TREE" \ + --arg parent "$START_SHA" \ + '{message: $message, tree: $tree, parents: [$parent]}' | + gh api "${REPOSITORY}/git/commits" -X POST --input - --jq '.sha' + ) + CURRENT_SHA=$(gh api "${REPOSITORY}/git/ref/heads/${BRANCH}" --jq '.object.sha') + if [ "$CURRENT_SHA" != "$START_SHA" ]; then + echo "Branch moved during generation; refusing to overwrite ${CURRENT_SHA}." >&2 + exit 1 fi + jq -n \ + --arg sha "$NEW_COMMIT" \ + '{sha: $sha, force: false}' | + gh api "${REPOSITORY}/git/refs/heads/${BRANCH}" -X PATCH --input - diff --git a/aks-node-controller/README.md b/aks-node-controller/README.md index 54c03523a6c..e262f146631 100644 --- a/aks-node-controller/README.md +++ b/aks-node-controller/README.md @@ -149,3 +149,31 @@ Key components: ``` This indicates the controller exited before emitting `provision.json`. Most commonly the rendered AKSNodeConfig was missing, had the wrong `Version` (expected `v1`), or was written to the wrong path (`/opt/azure/containers/aks-node-controller-config.json`). Fix the config generation, redeploy, and the bootstrap scripts will then populate `provision.json`. - **provision-wait**: waits for `provision.complete` to be present and reads `provision.json` which contains the provision output of type `CSEStatus` and is returned by CSE through capturing stdout. + +### Provisioning script hotfix payloads + +Patched ANC binaries can embed selected Linux provisioning scripts generated from +`parts/linux/cloud-init/artifacts/`. At the start of `provision`, ANC validates the +rendered nodecustomdata matching the local platform and atomically applies its +`write_files` entries before constructing the normal CSE command. +Application is fail-open so the existing VHD scripts remain usable if validation +or replacement fails. + +The ANC-owned `scripthotfix` package distinguishes these embedded script hotfixes +from updates to the ANC binary itself. The generated files live under +`aks-node-controller/scripthotfix/generated/` as +`rendered_nodecustomdata_.yml`. The generator selects only changed +hotfixable entries from `nodecustomdata.yml`, then renders Ubuntu, Mariner/Azure +Linux, ACL, OS Guard, and Flatcar variants through AgentBaker's production +Go-template functions. + +Embedded payloads are replace-only: ANC skips an entry when its runtime +destination does not already exist. File presence preserves non-platform +template gates such as custom-image exclusions. New-file hotfixes are not +supported by this delivery path. + +Script hotfix delivery is package-only. The existing base-to-version hotfix map +selects the ANC package for the node's baked `YYYYMM.DD` version base; the package +contains its corresponding rendered scripts. If the package cannot be installed, +provisioning fails open to the original VHD scripts. The operational fallback is +to upgrade the node image. diff --git a/aks-node-controller/app.go b/aks-node-controller/app.go index c44c513c660..4893b3807b9 100644 --- a/aks-node-controller/app.go +++ b/aks-node-controller/app.go @@ -22,6 +22,7 @@ import ( "github.com/Azure/agentbaker/aks-node-controller/parser" "github.com/Azure/agentbaker/aks-node-controller/pkg/gpu" "github.com/Azure/agentbaker/aks-node-controller/pkg/nodeconfigutils" + "github.com/Azure/agentbaker/aks-node-controller/scripthotfix" "github.com/fsnotify/fsnotify" "github.com/urfave/cli/v3" ) @@ -71,6 +72,8 @@ type App struct { // Authorization header for the check-hotfix LPS fetch. When nil, the real IMDS endpoint // is queried. fetchAttestedToken func(ctx context.Context) (string, error) + // applyEmbeddedHotfix overrides embedded script application for tests. + applyEmbeddedHotfix func(string) (scripthotfix.Result, error) // grpcDialContext overrides how the gRPC LPS client dials, letting tests point the client at // an in-process (bufconn) server. When nil, the real TLS dial to the apiserver front is used. grpcDialContext func(ctx context.Context, target string) (net.Conn, error) @@ -684,6 +687,23 @@ func (a *App) Provision(ctx context.Context, flags ProvisionFlags) (*ProvisionRe return provisionResult, err } +func (a *App) applyEmbeddedHotfixPayload() { + applyEmbeddedHotfix := a.applyEmbeddedHotfix + if applyEmbeddedHotfix == nil { + applyEmbeddedHotfix = scripthotfix.ApplyEmbedded + } + result, err := applyEmbeddedHotfix(a.osReleasePath) + if err != nil { + // Hotfixes are fail-open: the VHD-baked scripts remain available, so an + // embedded payload must not block provisioning. + slog.Warn("failed to apply embedded hotfix payload; continuing with existing scripts", + "error", err) + } else if result.Applied > 0 || result.Skipped > 0 { + slog.Info("processed embedded hotfix payload", + "applied", result.Applied, "skipped", result.Skipped) + } +} + // runProvision encapsulates execution for the "provision" subcommand after CLI parsing. // It returns an error describing any failure; callers should pass that error to // writeCompleteFileOnError so the sentinel file can be written on fail-fast paths. @@ -709,6 +729,8 @@ func (a *App) runProvision(ctx context.Context, flags ProvisionFlags, dryRun boo } if dryRun { a.cmdRun = cmdRunnerDryRun + } else { + a.applyEmbeddedHotfixPayload() } return a.Provision(ctx, flags) } diff --git a/aks-node-controller/app_test.go b/aks-node-controller/app_test.go index 1b11938f58b..114e89c5005 100644 --- a/aks-node-controller/app_test.go +++ b/aks-node-controller/app_test.go @@ -15,6 +15,7 @@ import ( "time" "github.com/Azure/agentbaker/aks-node-controller/helpers" + "github.com/Azure/agentbaker/aks-node-controller/scripthotfix" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -231,6 +232,70 @@ func TestApp_Run(t *testing.T) { } func TestApp_Provision(t *testing.T) { + t.Run("embedded hotfix runs before command construction and execution", func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + applied := false + tt.App.applyEmbeddedHotfix = func(string) (scripthotfix.Result, error) { + applied = true + return scripthotfix.Result{Applied: 1}, nil + } + + _, err := tt.App.runProvision( + context.Background(), + ProvisionFlags{ProvisionConfig: "does-not-exist.json"}, + false, + ) + + require.Error(t, err) + assert.True(t, applied, "embedded payload must run before config parsing") + }) + + t.Run("embedded hotfix failure is logged and provisioning continues", func(t *testing.T) { + logs := installLogCapturer(t) + executed := false + tt := NewTestApp(t, TestAppConfig{ + RunFunc: func(*exec.Cmd) error { + executed = true + return nil + }, + }) + tt.App.applyEmbeddedHotfix = func(string) (scripthotfix.Result, error) { + return scripthotfix.Result{}, errors.New("rendered nodecustomdata validation failed") + } + + _, err := tt.App.runProvision( + context.Background(), + ProvisionFlags{NBCCmd: "parser/testdata/test_nbccmd.sh"}, + false, + ) + + require.NoError(t, err) + assert.True(t, executed) + assert.Contains(t, logs.getRecords(), logRecord{ + Level: slog.LevelWarn, + Message: "failed to apply embedded hotfix payload; continuing with existing scripts", + Attrs: map[string]string{"error": "rendered nodecustomdata validation failed"}, + }) + }) + + t.Run("dry-run does not apply embedded hotfix payload", func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + applied := false + tt.App.applyEmbeddedHotfix = func(string) (scripthotfix.Result, error) { + applied = true + return scripthotfix.Result{}, nil + } + + _, err := tt.App.runProvision( + context.Background(), + ProvisionFlags{NBCCmd: "parser/testdata/test_nbccmd.sh"}, + true, + ) + + require.NoError(t, err) + assert.False(t, applied) + }) + t.Run("valid provision config", func(t *testing.T) { tt := NewTestApp(t, TestAppConfig{}) _, err := tt.App.Provision(context.Background(), ProvisionFlags{ProvisionConfig: "parser/testdata/test_aksnodeconfig.json"}) diff --git a/aks-node-controller/scripthotfix/applier.go b/aks-node-controller/scripthotfix/applier.go new file mode 100644 index 00000000000..ddfaae4e5fc --- /dev/null +++ b/aks-node-controller/scripthotfix/applier.go @@ -0,0 +1,422 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +// Package scripthotfix applies rendered provisioning script hotfixes embedded +// in the aks-node-controller binary. +package scripthotfix + +import ( + "bytes" + "compress/gzip" + "embed" + "encoding/base64" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "strconv" + "strings" + + "gopkg.in/yaml.v3" +) + +const defaultOSReleasePath = "/etc/os-release" + +type Platform string + +const ( + PlatformUbuntu Platform = "ubuntu" + PlatformMariner Platform = "mariner" + PlatformACL Platform = "acl" + PlatformAzlOSGuard Platform = "azlosguard" + PlatformFlatcar Platform = "flatcar" +) + +//go:embed generated +var embeddedGeneratedFiles embed.FS + +var generatedFiles fs.FS = embeddedGeneratedFiles + +type nodeCustomData struct { + WriteFiles []writeFile `yaml:"write_files"` +} + +type writeFile struct { + Path string `yaml:"path"` + Permissions string `yaml:"permissions"` + Encoding string `yaml:"encoding,omitempty"` + Owner string `yaml:"owner"` + Content string `yaml:"content"` +} + +type payloadEntry struct { + destination string + mode os.FileMode + content []byte +} + +type Result struct { + Applied int + Skipped int +} + +type stagedEntry struct { + destination string + stagedPath string + backupPath string + originalExist bool + preserveBackup bool +} + +// ApplyEmbedded applies the rendered payload compiled into this ANC binary. +func ApplyEmbedded(osReleasePath string) (Result, error) { + active, err := fs.ReadFile(generatedFiles, "generated/active") + if err != nil { + return Result{}, fmt.Errorf("read embedded hotfix state: %w", err) + } + if strings.TrimSpace(string(active)) != "true" { + return Result{}, nil + } + if osReleasePath == "" { + osReleasePath = defaultOSReleasePath + } + platform, err := ClassifyPlatform(osReleasePath) + if err != nil { + return Result{}, err + } + return applyFS(generatedFiles, platform) +} + +// ClassifyPlatform maps /etc/os-release to rendered nodecustomdata variants. +func ClassifyPlatform(osReleasePath string) (Platform, error) { + data, err := os.ReadFile(osReleasePath) + if err != nil { + return "", fmt.Errorf("read OS release %s: %w", osReleasePath, err) + } + values := parseOSRelease(data) + id := strings.ToLower(values["ID"]) + variant := strings.ToLower(values["VARIANT_ID"]) + + switch { + case variant == "osguard": + return PlatformAzlOSGuard, nil + case variant == "azurecontainerlinux", id == "azurecontainerlinux": + return PlatformACL, nil + case id == "ubuntu": + return PlatformUbuntu, nil + case id == "flatcar": + return PlatformFlatcar, nil + case id == "mariner", id == "azurelinux": + return PlatformMariner, nil + case id == "": + return "", fmt.Errorf("ID is missing from %s", osReleasePath) + default: + return "", fmt.Errorf("unsupported OS ID %q in %s", id, osReleasePath) + } +} + +func parseOSRelease(data []byte) map[string]string { + values := make(map[string]string) + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + key, value, found := strings.Cut(line, "=") + if !found { + continue + } + values[strings.TrimSpace(key)] = strings.Trim(strings.TrimSpace(value), `"'`) + } + return values +} + +func applyFS(payloadFS fs.FS, platform Platform) (Result, error) { + entries, err := loadAndValidate(payloadFS, platform) + if err != nil { + return Result{}, err + } + + result := Result{} + var staged []*stagedEntry + for _, entry := range entries { + pending, changed, err := stageEntry(entry.destination, entry.content, entry.mode) + if err != nil { + cleanupStaged(staged) + return result, fmt.Errorf("apply embedded hotfix to %s: %w", entry.destination, err) + } + if !changed { + result.Skipped++ + continue + } + staged = append(staged, &pending) + } + if err := commitStaged(staged); err != nil { + return Result{}, err + } + result.Applied = len(staged) + return result, nil +} + +func loadAndValidate(payloadFS fs.FS, platform Platform) ([]payloadEntry, error) { + if !isConcretePlatform(platform) { + return nil, fmt.Errorf("unsupported concrete platform %q", platform) + } + renderedPath := fmt.Sprintf( + "generated/rendered_nodecustomdata_%s.yml", + platform, + ) + data, err := fs.ReadFile(payloadFS, renderedPath) + if err != nil { + return nil, fmt.Errorf("read embedded nodecustomdata %s: %w", renderedPath, err) + } + + var customData nodeCustomData + decoder := yaml.NewDecoder(bytes.NewReader(data)) + decoder.KnownFields(true) + if err := decoder.Decode(&customData); err != nil { + return nil, fmt.Errorf("decode embedded nodecustomdata %s: %w", renderedPath, err) + } + var trailing any + if err := decoder.Decode(&trailing); err != io.EOF { + return nil, fmt.Errorf("embedded nodecustomdata %s has trailing content", renderedPath) + } + + entries := make([]payloadEntry, 0, len(customData.WriteFiles)) + destinations := make(map[string]struct{}, len(customData.WriteFiles)) + for index, file := range customData.WriteFiles { + entry, err := validateWriteFile(file) + if err != nil { + return nil, fmt.Errorf("validate embedded write_files entry %d: %w", index, err) + } + if _, exists := destinations[entry.destination]; exists { + return nil, fmt.Errorf("duplicate destination %s", entry.destination) + } + destinations[entry.destination] = struct{}{} + entries = append(entries, entry) + } + return entries, nil +} + +func validateWriteFile(file writeFile) (payloadEntry, error) { + if file.Path == "" || + (!strings.HasPrefix(file.Path, "/") && !filepath.IsAbs(file.Path)) || + (strings.HasPrefix(file.Path, "/") && path.Clean(file.Path) != file.Path) || + (!strings.HasPrefix(file.Path, "/") && filepath.Clean(file.Path) != file.Path) { + return payloadEntry{}, fmt.Errorf("unsafe destination %q", file.Path) + } + if strings.HasPrefix(file.Path, "/") && strings.Contains(file.Path, `\`) { + return payloadEntry{}, fmt.Errorf("unsafe destination %q: backslashes are not allowed", file.Path) + } + if file.Owner != "" && file.Owner != "root" { + return payloadEntry{}, fmt.Errorf("unsupported owner %q", file.Owner) + } + mode, err := parseMode(file.Permissions) + if err != nil { + return payloadEntry{}, err + } + content, err := decodeContent(file) + if err != nil { + return payloadEntry{}, err + } + if len(content) == 0 { + return payloadEntry{}, fmt.Errorf("content for %s is empty", file.Path) + } + return payloadEntry{ + destination: file.Path, + mode: mode, + content: content, + }, nil +} + +func parseMode(value string) (os.FileMode, error) { + parsed, err := strconv.ParseUint(value, 8, 32) + if err != nil || parsed == 0 || parsed > 0o777 { + return 0, fmt.Errorf("invalid mode %q", value) + } + return os.FileMode(parsed), nil +} + +func decodeContent(file writeFile) ([]byte, error) { + switch file.Encoding { + case "": + return []byte(file.Content), nil + case "base64": + decoded, err := base64.StdEncoding.DecodeString(file.Content) + if err != nil { + return nil, fmt.Errorf("decode base64 content for %s: %w", file.Path, err) + } + return decoded, nil + case "gzip": + reader, err := gzip.NewReader(bytes.NewReader([]byte(file.Content))) + if err != nil { + return nil, fmt.Errorf("create gzip reader for %s: %w", file.Path, err) + } + defer reader.Close() + decoded, err := io.ReadAll(reader) + if err != nil { + return nil, fmt.Errorf("read gzip content for %s: %w", file.Path, err) + } + return decoded, nil + default: + return nil, fmt.Errorf("unsupported encoding %q", file.Encoding) + } +} + +func isConcretePlatform(platform Platform) bool { + switch platform { + case PlatformUbuntu, PlatformMariner, PlatformACL, PlatformAzlOSGuard, PlatformFlatcar: + return true + default: + return false + } +} + +func stageEntry(destination string, payload []byte, mode os.FileMode) (stagedEntry, bool, error) { + current, err := os.ReadFile(destination) + var originalMode os.FileMode + switch { + case err == nil: + info, statErr := os.Stat(destination) + if statErr != nil { + return stagedEntry{}, false, fmt.Errorf("stat destination: %w", statErr) + } + originalMode = info.Mode().Perm() + if bytes.Equal(current, payload) && originalMode == mode.Perm() { + return stagedEntry{}, false, nil + } + case os.IsNotExist(err): + return stagedEntry{}, false, nil + default: + return stagedEntry{}, false, fmt.Errorf("read destination: %w", err) + } + + directory := filepath.Dir(destination) + info, err := os.Stat(directory) + if err != nil { + return stagedEntry{}, false, fmt.Errorf("stat destination directory %s: %w", directory, err) + } + if !info.IsDir() { + return stagedEntry{}, false, fmt.Errorf("destination parent %s is not a directory", directory) + } + + stagedPath, err := writeTempFile(directory, ".aks-node-controller-hotfix-stage-*", payload, mode) + if err != nil { + return stagedEntry{}, false, err + } + staged := stagedEntry{ + destination: destination, + stagedPath: stagedPath, + originalExist: true, + } + backupPath, err := writeTempFile( + directory, + ".aks-node-controller-hotfix-backup-*", + current, + originalMode, + ) + if err != nil { + _ = os.Remove(stagedPath) + return stagedEntry{}, false, fmt.Errorf("stage destination backup: %w", err) + } + staged.backupPath = backupPath + return staged, true, nil +} + +func writeTempFile(directory, pattern string, content []byte, mode os.FileMode) (string, error) { + temp, err := os.CreateTemp(directory, pattern) + if err != nil { + return "", fmt.Errorf("create temporary file: %w", err) + } + tempPath := temp.Name() + cleanup := func() { + _ = temp.Close() + _ = os.Remove(tempPath) + } + if _, err := temp.Write(content); err != nil { + cleanup() + return "", fmt.Errorf("write temporary file: %w", err) + } + if err := temp.Sync(); err != nil { + cleanup() + return "", fmt.Errorf("sync temporary file: %w", err) + } + if err := temp.Close(); err != nil { + cleanup() + return "", fmt.Errorf("close temporary file: %w", err) + } + if err := os.Chmod(tempPath, mode); err != nil { + cleanup() + return "", fmt.Errorf("chmod temporary file: %w", err) + } + return tempPath, nil +} + +func commitStaged(staged []*stagedEntry) error { + return commitStagedWithRename(staged, os.Rename) +} + +func commitStagedWithRename(staged []*stagedEntry, rename func(string, string) error) error { + committed := 0 + defer cleanupStaged(staged) + for index, entry := range staged { + if err := rename(entry.stagedPath, entry.destination); err != nil { + rollbackErr := rollbackStaged(staged[:committed], rename) + if rollbackErr != nil { + return fmt.Errorf( + "commit hotfix destination %s: %w; rollback failed: %w", + entry.destination, + err, + rollbackErr, + ) + } + return fmt.Errorf("commit hotfix destination %s: %w", entry.destination, err) + } + staged[index].stagedPath = "" + committed++ + } + return nil +} + +func rollbackStaged(committed []*stagedEntry, rename func(string, string) error) error { + var rollbackErrors []error + for index := len(committed) - 1; index >= 0; index-- { + entry := committed[index] + var err error + if entry.originalExist { + err = rename(entry.backupPath, entry.destination) + if err == nil { + entry.backupPath = "" + } + } else { + err = os.Remove(entry.destination) + } + if err != nil { + entry.preserveBackup = true + rollbackErrors = append( + rollbackErrors, + fmt.Errorf( + "restore %s from preserved backup %s: %w", + entry.destination, + entry.backupPath, + err, + ), + ) + } + } + return errors.Join(rollbackErrors...) +} + +func cleanupStaged(staged []*stagedEntry) { + for _, entry := range staged { + if entry.stagedPath != "" { + _ = os.Remove(entry.stagedPath) + } + if entry.backupPath != "" && !entry.preserveBackup { + _ = os.Remove(entry.backupPath) + } + } +} diff --git a/aks-node-controller/scripthotfix/applier_test.go b/aks-node-controller/scripthotfix/applier_test.go new file mode 100644 index 00000000000..dd3a2256847 --- /dev/null +++ b/aks-node-controller/scripthotfix/applier_test.go @@ -0,0 +1,301 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +package scripthotfix + +import ( + "bytes" + "compress/gzip" + "encoding/base64" + "errors" + "os" + "path/filepath" + "runtime" + "testing" + "testing/fstest" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +func TestClassifyPlatform(t *testing.T) { + tests := []struct { + name string + release string + expected Platform + }{ + {name: "Ubuntu", release: "ID=ubuntu\n", expected: PlatformUbuntu}, + {name: "Mariner", release: "ID=mariner\n", expected: PlatformMariner}, + {name: "Azure Linux", release: "ID=azurelinux\n", expected: PlatformMariner}, + { + name: "OS Guard variant wins over Azure Linux ID", + release: "ID=azurelinux\nVARIANT_ID=osguard\n", + expected: PlatformAzlOSGuard, + }, + { + name: "ACL variant wins over Azure Linux ID", + release: "ID=azurelinux\nVARIANT_ID=azurecontainerlinux\n", + expected: PlatformACL, + }, + {name: "ACL dedicated ID", release: "ID=azurecontainerlinux\n", expected: PlatformACL}, + {name: "Flatcar", release: "ID=flatcar\n", expected: PlatformFlatcar}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + releasePath := filepath.Join(t.TempDir(), "os-release") + require.NoError(t, os.WriteFile(releasePath, []byte(test.release), 0o600)) + + actual, err := ClassifyPlatform(releasePath) + + require.NoError(t, err) + assert.Equal(t, test.expected, actual) + }) + } + + t.Run("unsupported ID fails explicitly", func(t *testing.T) { + releasePath := filepath.Join(t.TempDir(), "os-release") + require.NoError(t, os.WriteFile(releasePath, []byte("ID=other\n"), 0o600)) + + _, err := ClassifyPlatform(releasePath) + + require.ErrorContains(t, err, "unsupported OS ID") + }) +} + +func TestApplyEmbeddedInactivePayloadDoesNotReadOSRelease(t *testing.T) { + original := generatedFiles + generatedFiles = fstest.MapFS{ + "generated/active": &fstest.MapFile{Data: []byte("false\n")}, + } + t.Cleanup(func() { + generatedFiles = original + }) + + result, err := ApplyEmbedded(filepath.Join(t.TempDir(), "missing-os-release")) + + require.NoError(t, err) + assert.Equal(t, Result{}, result) +} + +func TestApplyFSUsesSelectedRenderedNodeCustomData(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Windows rename cannot atomically replace an existing destination") + } + + directory := t.TempDir() + destination := filepath.Join(directory, "provision.sh") + require.NoError(t, os.WriteFile(destination, []byte("old"), 0o600)) + payload := []byte("#!/bin/sh\necho fixed\n") + files := renderedFS(t, PlatformUbuntu, []writeFile{{ + Path: destination, + Permissions: "0744", + Encoding: "base64", + Owner: "root", + Content: base64.StdEncoding.EncodeToString(payload), + }}) + + first, err := applyFS(files, PlatformUbuntu) + + require.NoError(t, err) + assert.Equal(t, Result{Applied: 1}, first) + actual, err := os.ReadFile(destination) + require.NoError(t, err) + assert.Equal(t, payload, actual) + info, err := os.Stat(destination) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o744), info.Mode().Perm()) + + second, err := applyFS(files, PlatformUbuntu) + + require.NoError(t, err) + assert.Equal(t, Result{Skipped: 1}, second) +} + +func TestApplyFSSkipsMissingDestination(t *testing.T) { + destination := filepath.Join(t.TempDir(), "missing.sh") + files := renderedFS(t, PlatformMariner, []writeFile{{ + Path: destination, + Permissions: "0744", + Owner: "root", + Content: "hotfix", + }}) + + result, err := applyFS(files, PlatformMariner) + + require.NoError(t, err) + assert.Equal(t, Result{Skipped: 1}, result) + _, statErr := os.Stat(destination) + assert.True(t, os.IsNotExist(statErr)) +} + +func TestRenderedNodeCustomDataValidation(t *testing.T) { + validDestination := filepath.Join(t.TempDir(), "provision.sh") + valid := writeFile{ + Path: validDestination, + Permissions: "0744", + Owner: "root", + Content: "hotfix", + } + + tests := []struct { + name string + files []writeFile + expectedErr string + }{ + { + name: "unsafe destination", + files: []writeFile{{ + Path: "../provision.sh", + Permissions: "0744", + Owner: "root", + Content: "hotfix", + }}, + expectedErr: "unsafe destination", + }, + { + name: "destination with embedded backslash", + files: []writeFile{{ + Path: `/tmp/provision\script.sh`, + Permissions: "0744", + Owner: "root", + Content: "hotfix", + }}, + expectedErr: "backslashes are not allowed", + }, + { + name: "invalid mode", + files: []writeFile{{ + Path: validDestination, + Permissions: "0999", + Owner: "root", + Content: "hotfix", + }}, + expectedErr: "invalid mode", + }, + { + name: "unsupported owner", + files: []writeFile{{ + Path: validDestination, + Permissions: "0744", + Owner: "nobody", + Content: "hotfix", + }}, + expectedErr: "unsupported owner", + }, + { + name: "duplicate destination", + files: []writeFile{valid, valid}, + expectedErr: "duplicate destination", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := loadAndValidate(renderedFS(t, PlatformUbuntu, test.files), PlatformUbuntu) + require.ErrorContains(t, err, test.expectedErr) + }) + } + + t.Run("unknown YAML field", func(t *testing.T) { + files := fstest.MapFS{ + renderedPath(PlatformUbuntu): &fstest.MapFile{ + Data: []byte("write_files: []\nunknown: true\n"), + }, + } + _, err := loadAndValidate(files, PlatformUbuntu) + require.ErrorContains(t, err, "field unknown not found") + }) +} + +func TestDecodeContentGzip(t *testing.T) { + var compressed bytes.Buffer + writer := gzip.NewWriter(&compressed) + _, err := writer.Write([]byte("rendered payload")) + require.NoError(t, err) + require.NoError(t, writer.Close()) + + decoded, err := decodeContent(writeFile{ + Encoding: "gzip", + Content: compressed.String(), + }) + + require.NoError(t, err) + assert.Equal(t, []byte("rendered payload"), decoded) +} + +func TestApplyFSDoesNotCommitWhenLaterEntryCannotBeStaged(t *testing.T) { + directory := t.TempDir() + firstDestination := filepath.Join(directory, "first.sh") + require.NoError(t, os.WriteFile(firstDestination, []byte("original"), 0o700)) + files := renderedFS(t, PlatformUbuntu, []writeFile{ + { + Path: firstDestination, + Permissions: "0744", + Owner: "root", + Content: "first hotfix", + }, + { + Path: directory, + Permissions: "0744", + Owner: "root", + Content: "second hotfix", + }, + }) + + _, err := applyFS(files, PlatformUbuntu) + + require.ErrorContains(t, err, "read destination") + actual, readErr := os.ReadFile(firstDestination) + require.NoError(t, readErr) + assert.Equal(t, []byte("original"), actual) +} + +func TestCommitStagedRollsBackEarlierReplacement(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Windows rename cannot atomically replace an existing destination") + } + directory := t.TempDir() + firstDestination := filepath.Join(directory, "first.sh") + secondDestination := filepath.Join(directory, "second.sh") + require.NoError(t, os.WriteFile(firstDestination, []byte("first original"), 0o700)) + require.NoError(t, os.WriteFile(secondDestination, []byte("second original"), 0o711)) + first, changed, err := stageEntry(firstDestination, []byte("first hotfix"), 0o744) + require.NoError(t, err) + require.True(t, changed) + second, changed, err := stageEntry(secondDestination, []byte("second hotfix"), 0o755) + require.NoError(t, err) + require.True(t, changed) + + err = commitStagedWithRename( + []*stagedEntry{&first, &second}, + func(source string, destination string) error { + if source == second.stagedPath { + return errors.New("injected rename failure") + } + return os.Rename(source, destination) + }, + ) + + require.ErrorContains(t, err, "injected rename failure") + firstActual, readErr := os.ReadFile(firstDestination) + require.NoError(t, readErr) + assert.Equal(t, []byte("first original"), firstActual) + secondActual, readErr := os.ReadFile(secondDestination) + require.NoError(t, readErr) + assert.Equal(t, []byte("second original"), secondActual) +} + +func renderedFS(t *testing.T, platform Platform, files []writeFile) fstest.MapFS { + t.Helper() + data, err := yaml.Marshal(nodeCustomData{WriteFiles: files}) + require.NoError(t, err) + return fstest.MapFS{ + renderedPath(platform): &fstest.MapFile{Data: data}, + } +} + +func renderedPath(platform Platform) string { + return "generated/rendered_nodecustomdata_" + string(platform) + ".yml" +} diff --git a/aks-node-controller/scripthotfix/generated/active b/aks-node-controller/scripthotfix/generated/active new file mode 100644 index 00000000000..c508d5366f7 --- /dev/null +++ b/aks-node-controller/scripthotfix/generated/active @@ -0,0 +1 @@ +false diff --git a/aks-node-controller/scripthotfix/generated/rendered_nodecustomdata_acl.yml b/aks-node-controller/scripthotfix/generated/rendered_nodecustomdata_acl.yml new file mode 100644 index 00000000000..7028abd713c --- /dev/null +++ b/aks-node-controller/scripthotfix/generated/rendered_nodecustomdata_acl.yml @@ -0,0 +1,2 @@ +#cloud-config +write_files: [] diff --git a/aks-node-controller/scripthotfix/generated/rendered_nodecustomdata_azlosguard.yml b/aks-node-controller/scripthotfix/generated/rendered_nodecustomdata_azlosguard.yml new file mode 100644 index 00000000000..7028abd713c --- /dev/null +++ b/aks-node-controller/scripthotfix/generated/rendered_nodecustomdata_azlosguard.yml @@ -0,0 +1,2 @@ +#cloud-config +write_files: [] diff --git a/aks-node-controller/scripthotfix/generated/rendered_nodecustomdata_flatcar.yml b/aks-node-controller/scripthotfix/generated/rendered_nodecustomdata_flatcar.yml new file mode 100644 index 00000000000..7028abd713c --- /dev/null +++ b/aks-node-controller/scripthotfix/generated/rendered_nodecustomdata_flatcar.yml @@ -0,0 +1,2 @@ +#cloud-config +write_files: [] diff --git a/aks-node-controller/scripthotfix/generated/rendered_nodecustomdata_mariner.yml b/aks-node-controller/scripthotfix/generated/rendered_nodecustomdata_mariner.yml new file mode 100644 index 00000000000..7028abd713c --- /dev/null +++ b/aks-node-controller/scripthotfix/generated/rendered_nodecustomdata_mariner.yml @@ -0,0 +1,2 @@ +#cloud-config +write_files: [] diff --git a/aks-node-controller/scripthotfix/generated/rendered_nodecustomdata_ubuntu.yml b/aks-node-controller/scripthotfix/generated/rendered_nodecustomdata_ubuntu.yml new file mode 100644 index 00000000000..7028abd713c --- /dev/null +++ b/aks-node-controller/scripthotfix/generated/rendered_nodecustomdata_ubuntu.yml @@ -0,0 +1,2 @@ +#cloud-config +write_files: [] diff --git a/e2e/scenario_test.go b/e2e/scenario_test.go index b55416f0629..2f53f9d8ddd 100644 --- a/e2e/scenario_test.go +++ b/e2e/scenario_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "os" "testing" "time" @@ -747,6 +748,72 @@ sudo "$anc_path" check-hotfix`, }) } +func Test_Ubuntu2204_EmbeddedScriptHotfix(t *testing.T) { + if config.Config.DisableScriptLessCompilation { + t.Skip("embedded script-hotfix E2E requires scriptless ANC compilation") + } + if config.Config.TestPreProvision { + t.Skip("embedded script-hotfix E2E does not run during two-stage VHD caching") + } + + const ( + runtimeScriptPath = "/opt/azure/containers/provision_configs.sh" + executionMarker = "/opt/azure/containers/e2e-script-hotfix-executed" + ) + marker := fmt.Sprintf("EMBEDDED_SCRIPT_HOTFIX_%d", time.Now().UnixNano()) + // Using the current script also exercises compatibility between the PR-built + // hotfix payload and the selected existing VHD's baked cse_main.sh. + payload, err := os.ReadFile("../parts/linux/cloud-init/artifacts/cse_config.sh") + if err != nil { + t.Fatalf("read hotfix payload: %v", err) + } + payload = append(payload, []byte(fmt.Sprintf( + "\nprintf '%%s\\n' '%s' > %s\n# %s\n", + marker, + executionMarker, + marker, + ))...) + + RunScenario(t, &Scenario{ + Description: "tests that a PR-built ANC applies an embedded script hotfix before provisioning", + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2204Gen2Containerd, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) { + }, + // The fixture intentionally replaces an older VHD's provision config + // with the current source, so broad source/VHD parity checks do not apply. + SkipDefaultValidation: true, + ScriptHotfixFixture: &ScriptHotfixFixture{ + Platform: "ubuntu", + Destination: runtimeScriptPath, + Mode: "0744", + Payload: payload, + }, + Validator: func(ctx context.Context, s *Scenario) error { + nodeName, err := s.Runtime.Kube.WaitUntilNodeReady(ctx, s.T, s.Runtime.VMSSName) + if err != nil { + return err + } + s.Runtime.VM.KubeName = nodeName + return errors.Join( + ValidateNodeCanRunAPod(ctx, s), + ValidateFileHasContent(ctx, s, runtimeScriptPath, marker), + ValidateFileHasContent(ctx, s, executionMarker, marker), + ValidateFileHasContent( + ctx, + s, + "/var/log/azure/aks-node-controller.output", + "processed embedded hotfix payload", + ), + ) + }, + }, + }) +} + func Test_Ubuntu2204FIPS(t *testing.T) { RunScenario(t, &Scenario{ Description: "Tests that a node using the Ubuntu 2204 FIPS Gen1 VHD can be properly bootstrapped", diff --git a/e2e/types.go b/e2e/types.go index 46902433bfe..06b29b2094e 100644 --- a/e2e/types.go +++ b/e2e/types.go @@ -194,6 +194,15 @@ type CustomDataWriteFile struct { Content string } +// ScriptHotfixFixture describes one script hotfix embedded into an isolated +// scenario-specific ANC build. +type ScriptHotfixFixture struct { + Platform string + Destination string + Mode string + Payload []byte +} + // Config represents the configuration of an AgentBaker E2E scenario. type Config struct { // Cluster creates, updates or re-uses an AKS cluster for the scenario @@ -230,6 +239,10 @@ type Config struct { // This is for e2e-only validation scenarios. CustomDataWriteFiles []CustomDataWriteFile + // ScriptHotfixFixture builds ANC in an isolated temporary module with this + // generated script-hotfix entry. It bypasses the shared ANC binary cache. + ScriptHotfixFixture *ScriptHotfixFixture + // Validator is a function where the scenario can perform any extra validation checks Validator func(ctx context.Context, s *Scenario) error diff --git a/e2e/vmss.go b/e2e/vmss.go index 676d4d05b59..55ce89d44b8 100644 --- a/e2e/vmss.go +++ b/e2e/vmss.go @@ -13,7 +13,9 @@ import ( "math/big" "os" "os/exec" + "path" "path/filepath" + "strconv" "strings" "testing" "time" @@ -26,17 +28,62 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v7" + "gopkg.in/yaml.v3" ) const ( loadBalancerBackendAddressPoolIDTemplate = "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadBalancers/kubernetes/backendAddressPools/aksOutboundBackendPool" ) +type scriptHotfixFixtureNodeCustomData struct { + WriteFiles []scriptHotfixFixtureWriteFile `yaml:"write_files"` +} + +type scriptHotfixFixtureWriteFile struct { + Path string `yaml:"path"` + Permissions string `yaml:"permissions"` + Encoding string `yaml:"encoding"` + Owner string `yaml:"owner"` + Content string `yaml:"content"` +} + func compileAndUploadAKSNodeController(ctx context.Context, arch string) (string, error) { binary, err := compileAKSNodeController(ctx, arch) if err != nil { return "", err } + defer binary.Close() + return uploadAKSNodeController(ctx, binary) +} + +func compileAndUploadAKSNodeControllerWithScriptHotfix( + ctx context.Context, + arch string, + fixture ScriptHotfixFixture, +) (string, error) { + buildDir, err := os.MkdirTemp("", "aks-node-controller-script-hotfix-*") + if err != nil { + return "", fmt.Errorf("create isolated ANC build directory: %w", err) + } + defer os.RemoveAll(buildDir) + + sourceDir := filepath.Join("..", "aks-node-controller") + if err := os.CopyFS(buildDir, os.DirFS(sourceDir)); err != nil { + return "", fmt.Errorf("copy ANC module for isolated script-hotfix build: %w", err) + } + if err := writeScriptHotfixFixture(buildDir, fixture); err != nil { + return "", err + } + + binary, err := compileAKSNodeControllerInDir(ctx, arch, buildDir) + if err != nil { + return "", err + } + defer binary.Close() + return uploadAKSNodeController(ctx, binary) +} + +func uploadAKSNodeController(ctx context.Context, binary *os.File) (string, error) { uniqueSuffix := randomLowercaseString(6) blobPath := fmt.Sprintf("%s/aks-node-controller-%s", time.Now().UTC().Format("2006-01-02-15-04-05"), uniqueSuffix) toolkit.Logf(ctx, "uploading aks-node-controller binary to blob path %s", blobPath) @@ -47,15 +94,22 @@ func compileAndUploadAKSNodeController(ctx context.Context, arch string) (string return url, nil } -// compileAndUploadAKSNodeController compiles the aks-node-controller binary for the given architecture. func compileAKSNodeController(ctx context.Context, arch string) (*os.File, error) { + return compileAKSNodeControllerInDir( + ctx, + arch, + filepath.Join("..", "aks-node-controller"), + ) +} + +func compileAKSNodeControllerInDir(ctx context.Context, arch, buildDir string) (*os.File, error) { goBin, err := exec.LookPath("go") if err != nil { return nil, fmt.Errorf("failed to find go binary in PATH: %w", err) } binName := "aks-node-controller-" + arch cmd := exec.CommandContext(ctx, goBin, "build", "-o", binName, "-v") - cmd.Dir = filepath.Join("..", "aks-node-controller") + cmd.Dir = buildDir cmd.Env = append(os.Environ(), "CGO_ENABLED=0", "GOOS=linux", @@ -66,13 +120,64 @@ func compileAKSNodeController(ctx context.Context, arch string) (*os.File, error if err != nil { return nil, fmt.Errorf("failed to compile aks-node-controller: %s", string(log)) } - f, err := os.Open(filepath.Join("..", "aks-node-controller", binName)) + f, err := os.Open(filepath.Join(buildDir, binName)) if err != nil { return nil, fmt.Errorf("failed to open compiled aks-node-controller binary: %w", err) } return f, nil } +func writeScriptHotfixFixture(buildDir string, fixture ScriptHotfixFixture) error { + if !path.IsAbs(fixture.Destination) || + path.Clean(fixture.Destination) != fixture.Destination || + strings.Contains(fixture.Destination, `\`) { + return fmt.Errorf("invalid script-hotfix fixture destination %q", fixture.Destination) + } + mode, err := strconv.ParseUint(fixture.Mode, 8, 32) + if err != nil || mode == 0 || mode > 0o777 { + return fmt.Errorf("invalid script-hotfix fixture mode %q", fixture.Mode) + } + validPlatforms := map[string]bool{ + "ubuntu": true, + "mariner": true, + "azlosguard": true, + "flatcar": true, + "acl": true, + } + if !validPlatforms[fixture.Platform] { + return fmt.Errorf("invalid script-hotfix fixture platform %q", fixture.Platform) + } + if len(fixture.Payload) == 0 { + return fmt.Errorf("script-hotfix fixture payload is empty") + } + + generatedDir := filepath.Join(buildDir, "scripthotfix", "generated") + rendered := scriptHotfixFixtureNodeCustomData{ + WriteFiles: []scriptHotfixFixtureWriteFile{{ + Path: fixture.Destination, + Permissions: fixture.Mode, + Encoding: "base64", + Owner: "root", + Content: base64.StdEncoding.EncodeToString(fixture.Payload), + }}, + } + data, err := yaml.Marshal(rendered) + if err != nil { + return fmt.Errorf("marshal rendered script-hotfix fixture: %w", err) + } + outputPath := filepath.Join( + generatedDir, + "rendered_nodecustomdata_"+fixture.Platform+".yml", + ) + if err := os.WriteFile(outputPath, data, 0o600); err != nil { + return fmt.Errorf("write rendered script-hotfix fixture: %w", err) + } + if err := os.WriteFile(filepath.Join(generatedDir, "active"), []byte("true\n"), 0o600); err != nil { + return fmt.Errorf("enable rendered script-hotfix fixture: %w", err) + } + return nil +} + // maxOutboundCSERetries bounds how many times node provisioning is retried when the // CSE outbound connectivity preflight check fails (ERR_OUTBOUND_CONN_FAIL / exit 50). // This is a known transient e2e-infrastructure flake; a genuine product regression @@ -261,7 +366,31 @@ func createVMSSModel(ctx context.Context, s *Scenario) (armcompute.VirtualMachin cse = nodeBootstrapping.CSE customData = nodeBootstrapping.CustomData - if enableScriptlessCompilation(s) { + if s.Config.ScriptHotfixFixture != nil { + if !enableScriptlessCompilation(s) { + return armcompute.VirtualMachineScaleSet{}, fmt.Errorf( + "script-hotfix fixture requires scriptless ANC compilation", + ) + } + binaryURL, err := compileAndUploadAKSNodeControllerWithScriptHotfix( + ctx, + s.VHD.Arch, + *s.Config.ScriptHotfixFixture, + ) + if err != nil { + return armcompute.VirtualMachineScaleSet{}, fmt.Errorf( + "compile and upload ANC with script-hotfix fixture: %w", + err, + ) + } + customData, err = CustomDataWithNBCCmdHack(customData, binaryURL) + if err != nil { + return armcompute.VirtualMachineScaleSet{}, fmt.Errorf( + "generate custom data with script-hotfix ANC: %w", + err, + ) + } + } else if enableScriptlessCompilation(s) { binaryURL, err := CachedCompileAndUploadAKSNodeController(ctx, s.VHD.Arch) if err != nil { return armcompute.VirtualMachineScaleSet{}, fmt.Errorf("compile and upload aks-node-controller binary: %w", err) diff --git a/e2e/vmss_test.go b/e2e/vmss_test.go index efc7aa7a11d..73efb37a30f 100644 --- a/e2e/vmss_test.go +++ b/e2e/vmss_test.go @@ -1,13 +1,94 @@ package e2e import ( + "encoding/base64" + "os" + "path/filepath" "testing" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v7" "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" ) +func TestWriteScriptHotfixFixture(t *testing.T) { + buildDir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(buildDir, "scripthotfix", "generated"), 0o755)) + fixture := ScriptHotfixFixture{ + Platform: "ubuntu", + Destination: "/opt/azure/containers/provision_configs.sh", + Mode: "0744", + Payload: []byte("#!/bin/bash\necho e2e\n"), + } + + require.NoError(t, writeScriptHotfixFixture(buildDir, fixture)) + + renderedData, err := os.ReadFile(filepath.Join( + buildDir, + "scripthotfix", + "generated", + "rendered_nodecustomdata_ubuntu.yml", + )) + require.NoError(t, err) + var rendered scriptHotfixFixtureNodeCustomData + require.NoError(t, yaml.Unmarshal(renderedData, &rendered)) + require.Len(t, rendered.WriteFiles, 1) + require.Equal(t, fixture.Destination, rendered.WriteFiles[0].Path) + require.Equal(t, fixture.Mode, rendered.WriteFiles[0].Permissions) + require.Equal(t, "base64", rendered.WriteFiles[0].Encoding) + payload, err := base64.StdEncoding.DecodeString(rendered.WriteFiles[0].Content) + require.NoError(t, err) + require.Equal(t, fixture.Payload, payload) +} + +func TestWriteScriptHotfixFixtureRejectsInvalidData(t *testing.T) { + valid := ScriptHotfixFixture{ + Platform: "ubuntu", + Destination: "/opt/azure/containers/provision_configs.sh", + Mode: "0744", + Payload: []byte("#!/bin/bash\n"), + } + tests := []struct { + name string + mutate func(*ScriptHotfixFixture) + }{ + { + name: "relative destination", + mutate: func(fixture *ScriptHotfixFixture) { + fixture.Destination = "opt/provision_configs.sh" + }, + }, + { + name: "invalid mode", + mutate: func(fixture *ScriptHotfixFixture) { + fixture.Mode = "0999" + }, + }, + { + name: "unsupported platform", + mutate: func(fixture *ScriptHotfixFixture) { + fixture.Platform = "other" + }, + }, + { + name: "empty payload", + mutate: func(fixture *ScriptHotfixFixture) { + fixture.Payload = nil + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fixture := valid + test.mutate(&fixture) + buildDir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(buildDir, "scripthotfix", "generated"), 0o755)) + require.Error(t, writeScriptHotfixFixture(buildDir, fixture)) + }) + } +} + // TestCSEExitCodeOutboundConnFail pins the exit code constant to the value emitted by // ERR_OUTBOUND_CONN_FAIL in parts/linux/cloud-init/artifacts/cse_helpers.sh. If the // product error code changes, this test forces the harness mitigation to be updated. diff --git a/hotfix/hotfix_generate.py b/hotfix/hotfix_generate.py index 48f58d0b72c..2b673e648d1 100644 --- a/hotfix/hotfix_generate.py +++ b/hotfix/hotfix_generate.py @@ -8,16 +8,13 @@ testdata files vs the base branch, bumps the patch of the current pkg/agent/datamodel/linux_sig_version.json version and uses it as `version`. -2. Detects which CSE provisioning scripts changed vs the base branch and injects their - write_files entries into the EnableScriptlessCSECmd section of - parts/linux/cloud-init/nodecustomdata.yml. If that injection (or a direct edit) - leaves nodecustomdata.yml different from the base branch, `scripts_version` is - bumped using the same base-version + tag-collision algorithm as `version`. +2. Detects which CSE provisioning scripts changed vs the base branch, selects their + write_files entries from parts/linux/cloud-init/nodecustomdata.yml, and renders + self-contained ANC payloads for each Linux platform with AgentBaker's canonical + Go-template renderer. -3. Writes the resulting {"version", "scripts_version"} (omitting fields that don't - apply) to parts/linux/cloud-init/artifacts/aks-node-controller-hotfix.json only - when there is an active hotfix. If no hotfix applies, the file is removed/left - absent so scriptless customData does not embed an empty hotfix artifact. +3. Writes the resolved ANC `version` to + parts/linux/cloud-init/artifacts/aks-node-controller-hotfix.json when active. Usage: python3 hotfix/hotfix_generate.py base_ref: git ref to diff against for changed-script/changed-code detection @@ -26,9 +23,11 @@ This script is called by the hotfix-generate GH Action. """ +import argparse import json import os import re +import shutil import subprocess import sys @@ -37,13 +36,10 @@ ARTIFACTS_DIR = "parts/linux/cloud-init/artifacts" LINUX_SIG_VERSION_FILE = "pkg/agent/datamodel/linux_sig_version.json" ANC_DIR = "aks-node-controller/" +GENERATED_DIR = os.path.join(ANC_DIR, "scripthotfix", "generated") VERSION_RE = re.compile(r'^\d{6}\.\d{2}\.\d+$') -# Marker comments for idempotent injection of the raw changed-script blocks. -SCRIPTS_BEGIN = "# ---- hotfix-scripts: auto-generated ----" -SCRIPTS_END = "# ---- end hotfix-scripts ----" - # Map from source file paths (relative to artifacts/) to the GetVariableProperty # keys used in nodecustomdata.yml. Only scripts that appear as write_files entries # in the traditional section are included. @@ -84,6 +80,7 @@ "validate-kubelet-credentials.sh": "validateKubeletCredentialsScript", "setup-custom-search-domains.sh": "customSearchDomainsScript", "configure-azure-network.sh": "configureAzureNetworkScript", + "init-aks-custom-cloud.sh": "initAKSCustomCloud", "init-aks-cloud.sh": "initAKSCloud", # Distro-specific scripts "ubuntu/ubuntu-snapshot-update.sh": "snapshotUpdateScript", @@ -121,6 +118,33 @@ "provisionInstallsACL": "install_distro", } +VARKEY_TO_SOURCE = {varkey: source for source, varkey in SOURCE_TO_VARKEY.items()} + +HOTFIXABLE_SUFFIXES = ( + ".sh", + ".py", + ".service", + ".timer", + ".rules", +) +GENERATED_ARTIFACTS = { + "aks-node-controller-hotfix.json", +} + +class GenerationError(RuntimeError): + """Raised when hotfix assets cannot be generated safely.""" + + +def validate_source_mappings(): + """Validate the explicit hotfixable source allowlist.""" + if len(VARKEY_TO_SOURCE) != len(SOURCE_TO_VARKEY): + raise GenerationError("source mappings contain duplicate variable keys") + for varkey in VARKEY_TO_BLOCK_GROUP: + if varkey not in VARKEY_TO_SOURCE: + raise GenerationError( + f"distro block variable key {varkey} has no source mapping" + ) + def read_base_version(): """Read the current released VHD image version, e.g. '202607.02.0'.""" @@ -170,8 +194,8 @@ def path_changed(base_ref, *paths): raise subprocess.CalledProcessError(result.returncode, result.args) -def write_hotfix_file(version, scripts_version): - """Write the resolved {version, scripts_version} to TARGET_FILE when active. +def write_hotfix_file(version): + """Write the resolved ANC version to TARGET_FILE when active. When no hotfix applies, remove TARGET_FILE if present. An empty JSON object is still embedded as a real scriptless customData file, which changes payload @@ -180,8 +204,6 @@ def write_hotfix_file(version, scripts_version): payload = {} if version: payload["version"] = version - if scripts_version: - payload["scripts_version"] = scripts_version if payload: with open(TARGET_FILE, "w") as f: @@ -197,7 +219,7 @@ def write_hotfix_file(version, scripts_version): print(f"No active hotfix; {TARGET_FILE} already absent", file=sys.stderr) -def detect_changed_varkeys(base_ref): +def detect_changed_varkeys(base_ref, available_varkeys=None): """Detect changed scripts via git diff and return the set of varkeys to inject.""" result = subprocess.run( ["git", "diff", "--name-only", base_ref, "--", f"{ARTIFACTS_DIR}/"], @@ -217,12 +239,23 @@ def detect_changed_varkeys(base_ref): for filepath in changed_files.splitlines(): local_path = filepath.removeprefix(f"{ARTIFACTS_DIR}/") + if local_path in GENERATED_ARTIFACTS: + continue if local_path in SOURCE_TO_VARKEY: + source_path = os.path.join(ARTIFACTS_DIR, local_path) + if not os.path.isfile(source_path): + raise GenerationError( + f"changed hotfix source {local_path} does not exist at {source_path}" + ) varkey = SOURCE_TO_VARKEY[local_path] matched_varkeys.add(varkey) if varkey in VARKEY_TO_BLOCK_GROUP: matched_block_groups.add(VARKEY_TO_BLOCK_GROUP[varkey]) print(f" Matched: {local_path} → {varkey}") + elif local_path.endswith(HOTFIXABLE_SUFFIXES) or local_path == "manifest.json": + raise GenerationError( + f"changed hotfixable artifact {local_path} has no source/runtime mapping" + ) else: print(f" Warning: {local_path} has no mapping in SOURCE_TO_VARKEY (skipped)") @@ -232,9 +265,22 @@ def detect_changed_varkeys(base_ref): # If a distro block group was matched, add all members of that group for varkey, group in VARKEY_TO_BLOCK_GROUP.items(): - if group in matched_block_groups: + if ( + group in matched_block_groups + and (available_varkeys is None or varkey in available_varkeys) + ): matched_varkeys.add(varkey) + for varkey in matched_varkeys: + source = VARKEY_TO_SOURCE.get(varkey) + if not source: + raise GenerationError(f"variable key {varkey} has no source mapping") + source_path = os.path.join(ARTIFACTS_DIR, source) + if not os.path.isfile(source_path): + raise GenerationError( + f"selected hotfix source {source} does not exist at {source_path}" + ) + print(f"\nVariable keys to inject: {' '.join(sorted(matched_varkeys))}") return matched_varkeys @@ -249,17 +295,29 @@ def find_block_boundaries(lines): stripped = line.strip() if '{{if EnableScriptlessCSECmd}}' in stripped or '{{ if EnableScriptlessCSECmd }}' in stripped: scriptless_start = i - elif scriptless_start is not None and else_line is None and stripped.startswith('{{- else'): - else_line = i + break - for i in range(len(lines) - 1, -1, -1): + if scriptless_start is None: + return None, None, None + + depth = 0 + for i in range(scriptless_start, len(lines)): stripped = lines[i].strip() + if re.match(r'\{\{-?\s*if(?:\s+|$)', stripped): + depth += 1 + continue + if ( + depth == 1 + and else_line is None + and re.match(r'\{\{-?\s*else\s*-?\}\}$', stripped) + ): + else_line = i + continue if re.match(r'\{\{-?\s*end\s*-?\}\}$', stripped): - end_line = i - break - - if else_line is not None and end_line is not None and end_line <= else_line: - end_line = None + depth -= 1 + if depth == 0: + end_line = i + break return scriptless_start, else_line, end_line @@ -319,97 +377,91 @@ def parse_write_files_blocks(traditional_lines): return blocks -def remove_scripts_block(): - """Remove any previously injected hotfix-scripts block (idempotent cleanup).""" - with open(TEMPLATE) as f: - content = f.read() - - new_content = re.sub( - rf'\n?{re.escape(SCRIPTS_BEGIN)}\n.*?{re.escape(SCRIPTS_END)}\n', - '', content, flags=re.DOTALL, - ) - - if new_content != content: - with open(TEMPLATE, 'w') as f: - f.write(new_content) - print(f"Removed previous hotfix-scripts block from {TEMPLATE}", file=sys.stderr) - return True - return False - - -def inject_scripts(target_varkeys): - """Extract matching write_files blocks from the traditional section and inject - them into the scriptless section, replacing any previously injected block.""" - with open(TEMPLATE, 'r') as f: - content = f.read() - - content = re.sub( - rf'\n?{re.escape(SCRIPTS_BEGIN)}\n.*?{re.escape(SCRIPTS_END)}\n', - '', content, flags=re.DOTALL, - ) - - lines = content.splitlines(keepends=True) - - scriptless_start, else_line, end_line = find_block_boundaries(lines) - if scriptless_start is None or else_line is None or end_line is None: - print("ERROR: Could not find EnableScriptlessCSECmd block boundaries", file=sys.stderr) - print(f" scriptless_start={scriptless_start}, else_line={else_line}, end_line={end_line}", file=sys.stderr) - sys.exit(1) - - print("\nTemplate structure:", file=sys.stderr) - print(f" EnableScriptlessCSECmd block: lines {scriptless_start+1}-{else_line+1}", file=sys.stderr) - print(f" Traditional block: lines {else_line+2}-{end_line+1}", file=sys.stderr) - - traditional_lines = lines[else_line+1:end_line] +def build_hotfix_template(target_varkeys, traditional_lines): + """Build a hotfix-only nodecustomdata template from canonical write_files blocks.""" blocks = parse_write_files_blocks(traditional_lines) - print(f"Found {len(blocks)} write_files blocks in traditional section", file=sys.stderr) - selected_blocks = [] for varkeys, block_lines in blocks: if varkeys & target_varkeys: selected_blocks.append(block_lines) - print(f" Selected block with varkeys: {varkeys}", file=sys.stderr) + if target_varkeys and not selected_blocks: + raise GenerationError("no matching write_files blocks found") if not selected_blocks: - print("No matching write_files blocks found for the target varkeys.", file=sys.stderr) - return False + return "#cloud-config\nwrite_files: []\n" - scripts_lines = [ - "\n", - f"{SCRIPTS_BEGIN}\n", - ] + rendered = ["#cloud-config\n", "write_files:\n"] for block_lines in selected_blocks: - scripts_lines.extend(block_lines) - scripts_lines.append(f"{SCRIPTS_END}\n") + rendered.extend(block_lines) + return "".join(rendered) - final_lines = lines[:else_line] + scripts_lines + lines[else_line:] - with open(TEMPLATE, 'w') as f: - f.writelines(final_lines) +def write_rendered_payload(target_varkeys, traditional_lines): + """Render platform-specific YAML through AgentBaker's production template path.""" + hotfix_template = build_hotfix_template(target_varkeys, traditional_lines) + shutil.rmtree(GENERATED_DIR, ignore_errors=True) + os.makedirs(GENERATED_DIR, exist_ok=True) - print(f"\nInjected {len(selected_blocks)} write_files block(s) into EnableScriptlessCSECmd section", file=sys.stderr) - print(f"Updated {TEMPLATE}", file=sys.stderr) - return True + template_path = os.path.join(GENERATED_DIR, ".nodecustomdata-hotfix.template") + with open(template_path, "w", newline="\n") as template_file: + template_file.write(hotfix_template) + try: + subprocess.run( + [ + "go", + "run", + "./hotfix/render-nodecustomdata", + "--template", + template_path, + "--output-dir", + GENERATED_DIR, + ], + check=True, + ) + finally: + try: + os.remove(template_path) + except FileNotFoundError: + pass + with open(os.path.join(GENERATED_DIR, "active"), "w", newline="\n") as active_file: + active_file.write("true\n" if target_varkeys else "false\n") + print( + f"Rendered {len(target_varkeys)} hotfix variable keys into {GENERATED_DIR}", + file=sys.stderr, + ) def main(): - if len(sys.argv) < 2: - print("Usage: python3 hotfix/hotfix_generate.py ", file=sys.stderr) - sys.exit(1) - base_ref = sys.argv[1] + parser = argparse.ArgumentParser(description="Generate ANC hotfix assets") + parser.add_argument("base_ref", help="git ref to diff against") + args = parser.parse_args() + base_ref = args.base_ref # Best-effort: make sure locally-known tags are up to date before checking for # collisions. Ignore failures (e.g. no network) and fall back to local tags. subprocess.run(["git", "fetch", "--tags"], capture_output=True) - # Detect & inject changed CSE scripts into nodecustomdata.yml first, since whether - # that leaves the template modified is itself the signal used below to decide - # scripts_version. - target_varkeys = detect_changed_varkeys(base_ref) - if target_varkeys: - inject_scripts(target_varkeys) - else: - remove_scripts_block() + try: + validate_source_mappings() + with open(TEMPLATE, "r") as template_file: + template_lines = template_file.readlines() + _, else_line, end_line = find_block_boundaries(template_lines) + if else_line is None or end_line is None: + raise GenerationError( + f"could not find traditional write_files section in {TEMPLATE}" + ) + traditional_lines = template_lines[else_line + 1:end_line] + available_varkeys = set() + for varkeys, _ in parse_write_files_blocks(traditional_lines): + available_varkeys.update(varkeys) + changed_varkeys = detect_changed_varkeys( + base_ref, + available_varkeys=available_varkeys, + ) + write_rendered_payload(changed_varkeys, traditional_lines) + except GenerationError as err: + print(f"ERROR: {err}", file=sys.stderr) + sys.exit(1) base_version = read_base_version() @@ -427,14 +479,7 @@ def main(): print(f"aks-node-controller/ has no production changes vs {base_ref}; " "version not set", file=sys.stderr) - scripts_version = "" - if path_changed(base_ref, TEMPLATE): - scripts_version = bump_version(base_version) - print(f"{TEMPLATE} changed vs {base_ref}; scripts_version={scripts_version}", file=sys.stderr) - else: - print(f"{TEMPLATE} unchanged vs {base_ref}; scripts_version not set", file=sys.stderr) - - write_hotfix_file(version, scripts_version) + write_hotfix_file(version) if __name__ == '__main__': diff --git a/hotfix/hotfix_generate_test.py b/hotfix/hotfix_generate_test.py new file mode 100644 index 00000000000..0ee23e37cd9 --- /dev/null +++ b/hotfix/hotfix_generate_test.py @@ -0,0 +1,238 @@ +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest import mock + +from hotfix import hotfix_generate + + +TRADITIONAL_TEMPLATE = """- path: {{GetCSEHelpersScriptFilepath}} + permissions: "0744" + encoding: gzip + owner: root + content: !!binary | + {{GetVariableProperty "cloudInitData" "provisionSource"}} +{{if IsACL }} +- path: {{GetCSEHelpersScriptDistroFilepath}} + permissions: "0744" + encoding: gzip + owner: root + content: !!binary | + {{GetVariableProperty "cloudInitData" "provisionSourceACL"}} +{{- else if IsAzlOSGuard}} +- path: {{GetCSEHelpersScriptDistroFilepath}} + permissions: "0744" + encoding: gzip + owner: root + content: !!binary | + {{GetVariableProperty "cloudInitData" "provisionSourceAzlOSGuard"}} +{{- else if IsMariner}} +- path: {{GetCSEHelpersScriptDistroFilepath}} + permissions: "0744" + encoding: gzip + owner: root + content: !!binary | + {{GetVariableProperty "cloudInitData" "provisionSourceMariner"}} +{{- else if IsFlatcar }} +- path: {{GetCSEHelpersScriptDistroFilepath}} + permissions: "0744" + encoding: gzip + owner: root + content: !!binary | + {{GetVariableProperty "cloudInitData" "provisionSourceFlatcar"}} +{{- else }} +- path: {{GetCSEHelpersScriptDistroFilepath}} + permissions: "0744" + encoding: gzip + owner: root + content: !!binary | + {{GetVariableProperty "cloudInitData" "provisionSourceUbuntu"}} +{{end}} +""" + + +class HotfixGenerateTests(unittest.TestCase): + def test_find_block_boundaries(self): + content = f"""#cloud-config +write_files: +{{{{if EnableScriptlessCSECmd}}}} +{{{{- else }}}} +{TRADITIONAL_TEMPLATE}{{{{- end }}}} +""" + + start, outer_else, end = hotfix_generate.find_block_boundaries( + content.splitlines(keepends=True) + ) + + self.assertEqual(2, start) + self.assertEqual(3, outer_else) + self.assertEqual(len(content.splitlines()) - 1, end) + + def test_parse_write_files_blocks_keeps_distro_chain_together(self): + blocks = hotfix_generate.parse_write_files_blocks( + TRADITIONAL_TEMPLATE.splitlines(keepends=True) + ) + + self.assertEqual(2, len(blocks)) + self.assertEqual({"provisionSource"}, blocks[0][0]) + self.assertEqual( + { + "provisionSourceUbuntu", + "provisionSourceMariner", + "provisionSourceAzlOSGuard", + "provisionSourceFlatcar", + "provisionSourceACL", + }, + blocks[1][0], + ) + + def test_build_hotfix_template_selects_only_requested_blocks(self): + rendered = hotfix_generate.build_hotfix_template( + {"provisionSource"}, + TRADITIONAL_TEMPLATE.splitlines(keepends=True), + ) + + self.assertTrue(rendered.startswith("#cloud-config\nwrite_files:\n")) + self.assertIn("provisionSource", rendered) + self.assertNotIn("provisionSourceUbuntu", rendered) + + def test_build_hotfix_template_emits_valid_empty_document(self): + rendered = hotfix_generate.build_hotfix_template( + set(), + TRADITIONAL_TEMPLATE.splitlines(keepends=True), + ) + + self.assertEqual("#cloud-config\nwrite_files: []\n", rendered) + + def test_detect_changed_varkeys_expands_distro_group(self): + with tempfile.TemporaryDirectory() as temp_dir: + artifacts = Path(temp_dir) + for source in ( + "ubuntu/cse_helpers_ubuntu.sh", + "mariner/cse_helpers_mariner.sh", + "azlosguard/cse_helpers_osguard.sh", + "flatcar/cse_helpers_flatcar.sh", + "acl/cse_helpers_acl.sh", + ): + path = artifacts / source + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("hotfix") + changed = artifacts / "ubuntu/cse_helpers_ubuntu.sh" + result = subprocess.CompletedProcess( + args=[], + returncode=0, + stdout=f"{changed}\n", + ) + available = { + "provisionSourceUbuntu", + "provisionSourceMariner", + "provisionSourceAzlOSGuard", + "provisionSourceFlatcar", + "provisionSourceACL", + } + + with mock.patch.object( + hotfix_generate, "ARTIFACTS_DIR", temp_dir + ), mock.patch.object( + hotfix_generate.subprocess, "run", return_value=result + ): + selected = hotfix_generate.detect_changed_varkeys( + "base", + available_varkeys=available, + ) + + self.assertEqual(available, selected) + + def test_write_rendered_payload_uses_canonical_renderer(self): + with tempfile.TemporaryDirectory() as temp_dir: + generated = Path(temp_dir) / "generated" + + def render(command, check): + self.assertTrue(check) + self.assertEqual("go", command[0]) + output_dir = Path(command[command.index("--output-dir") + 1]) + for platform in ( + "ubuntu", + "mariner", + "acl", + "azlosguard", + "flatcar", + ): + (output_dir / f"rendered_nodecustomdata_{platform}.yml").write_text( + "#cloud-config\n" + "write_files:\n" + "- path: /opt/azure/containers/provision_source.sh\n" + " permissions: \"0744\"\n" + " owner: root\n" + " content: hotfix\n" + ) + + with mock.patch.object( + hotfix_generate, "GENERATED_DIR", str(generated) + ), mock.patch.object( + hotfix_generate.subprocess, "run", side_effect=render + ): + hotfix_generate.write_rendered_payload( + {"provisionSource"}, + TRADITIONAL_TEMPLATE.splitlines(keepends=True), + ) + + expected = { + "ubuntu", + "mariner", + "acl", + "azlosguard", + "flatcar", + } + actual = { + path.name.removeprefix("rendered_nodecustomdata_").removesuffix(".yml") + for path in generated.glob("rendered_nodecustomdata_*.yml") + } + self.assertEqual(expected, actual) + for path in generated.glob("rendered_nodecustomdata_*.yml"): + content = path.read_text() + self.assertIn("/opt/azure/containers/provision_source.sh", content) + self.assertNotIn("{{", content) + self.assertFalse((generated / ".nodecustomdata-hotfix.template").exists()) + self.assertEqual("true\n", (generated / "active").read_text()) + + def test_write_hotfix_file_contains_only_anc_version(self): + with tempfile.TemporaryDirectory() as temp_dir: + target = Path(temp_dir) / "hotfix.json" + with mock.patch.object( + hotfix_generate, "TARGET_FILE", str(target) + ): + hotfix_generate.write_hotfix_file("202608.14.1") + self.assertEqual( + {"version": "202608.14.1"}, + json.loads(target.read_text()), + ) + hotfix_generate.write_hotfix_file("") + self.assertFalse(target.exists()) + + def test_unmapped_hotfixable_script_fails(self): + with tempfile.TemporaryDirectory() as temp_dir: + changed = Path(temp_dir) / "unmapped.sh" + changed.write_text("#!/bin/sh") + result = subprocess.CompletedProcess( + args=[], + returncode=0, + stdout=f"{changed}\n", + ) + with mock.patch.object( + hotfix_generate, "ARTIFACTS_DIR", temp_dir + ), mock.patch.object( + hotfix_generate.subprocess, "run", return_value=result + ): + with self.assertRaisesRegex( + hotfix_generate.GenerationError, + "has no source/runtime mapping", + ): + hotfix_generate.detect_changed_varkeys("base") + + +if __name__ == "__main__": + unittest.main() diff --git a/hotfix/render-nodecustomdata/main.go b/hotfix/render-nodecustomdata/main.go new file mode 100644 index 00000000000..f9232ea0a58 --- /dev/null +++ b/hotfix/render-nodecustomdata/main.go @@ -0,0 +1,98 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +package main + +import ( + "flag" + "fmt" + "os" + "path/filepath" + + "github.com/Azure/agentbaker/pkg/agent" + "github.com/Azure/agentbaker/pkg/agent/datamodel" +) + +type platform struct { + name string + distro datamodel.Distro +} + +var platforms = []platform{ + {name: "ubuntu", distro: datamodel.AKSUbuntuContainerd2204Gen2}, + {name: "mariner", distro: datamodel.AKSAzureLinuxV3Gen2}, + {name: "acl", distro: datamodel.AKSACLGen2TL}, + {name: "azlosguard", distro: datamodel.AKSAzureLinuxV3OSGuardGen2FIPSTL}, + {name: "flatcar", distro: datamodel.AKSFlatcarGen2}, +} + +func main() { + templatePath := flag.String("template", "", "path to the hotfix nodecustomdata template") + outputDir := flag.String("output-dir", "", "directory for rendered nodecustomdata files") + flag.Parse() + + if *templatePath == "" || *outputDir == "" { + fmt.Fprintln(os.Stderr, "--template and --output-dir are required") + os.Exit(2) + } + + templateContent, err := os.ReadFile(*templatePath) + if err != nil { + fatalf("read template: %v", err) + } + if err := os.MkdirAll(*outputDir, 0o755); err != nil { + fatalf("create output directory: %v", err) + } + + for _, target := range platforms { + rendered, err := agent.RenderLinuxNodeCustomDataTemplate( + templateContent, + newRenderConfig(target.distro), + ) + if err != nil { + fatalf("render %s nodecustomdata: %v", target.name, err) + } + outputPath := filepath.Join( + *outputDir, + "rendered_nodecustomdata_"+target.name+".yml", + ) + if err := os.WriteFile(outputPath, []byte(rendered), 0o644); err != nil { + fatalf("write %s nodecustomdata: %v", target.name, err) + } + } +} + +func newRenderConfig(distro datamodel.Distro) *datamodel.NodeBootstrappingConfiguration { + profile := &datamodel.AgentPoolProfile{ + Name: "hotfix-render", + OSType: datamodel.Linux, + Distro: distro, + } + return &datamodel.NodeBootstrappingConfiguration{ + ContainerService: &datamodel.ContainerService{ + Location: "eastus", + Properties: &datamodel.Properties{ + OrchestratorProfile: &datamodel.OrchestratorProfile{ + OrchestratorVersion: "1.29.0", + OrchestratorType: datamodel.Kubernetes, + KubernetesConfig: &datamodel.KubernetesConfig{ + ContainerRuntimeConfig: map[string]string{}, + }, + }, + HostedMasterProfile: &datamodel.HostedMasterProfile{ + FQDN: "hotfix-render.invalid", + }, + AgentPoolProfiles: []*datamodel.AgentPoolProfile{profile}, + }, + }, + AgentPoolProfile: profile, + CloudSpecConfig: datamodel.AzurePublicCloudSpecForTest, + K8sComponents: &datamodel.K8sComponents{}, + KubeletConfig: map[string]string{}, + } +} + +func fatalf(format string, args ...interface{}) { + fmt.Fprintf(os.Stderr, format+"\n", args...) + os.Exit(1) +} diff --git a/pkg/agent/baker.go b/pkg/agent/baker.go index 4cfe047363e..4334fe55071 100644 --- a/pkg/agent/baker.go +++ b/pkg/agent/baker.go @@ -632,6 +632,29 @@ func (t *TemplateGenerator) getSingleLine(textFilename string, profile interface return expandedTemplate, nil } +// RenderLinuxNodeCustomDataTemplate renders a nodecustomdata template with the +// same variables and functions used by the production AgentBaker path. +func RenderLinuxNodeCustomDataTemplate(templateContent []byte, config *datamodel.NodeBootstrappingConfiguration) (string, error) { + if config == nil || config.AgentPoolProfile == nil || config.ContainerService == nil || config.ContainerService.Properties == nil { + return "", fmt.Errorf("node bootstrapping configuration is incomplete") + } + + parameters := getParameters(config) + variables := getCustomDataVariables(config) + templ := template.New("nodecustomdata template"). + Option("missingkey=zero"). + Funcs(getBakerFuncMap(config, parameters, variables)) + if _, err := templ.Parse(string(removeComments(templateContent))); err != nil { + return "", fmt.Errorf("error parsing nodecustomdata template: %w", err) + } + + var buffer bytes.Buffer + if err := templ.Execute(&buffer, config.AgentPoolProfile); err != nil { + return "", fmt.Errorf("error executing nodecustomdata template: %w", err) + } + return buffer.String(), nil +} + // getTemplateFuncMap returns the general purpose template func map from getContainerServiceFuncMap. func getBakerFuncMap(config *datamodel.NodeBootstrappingConfiguration, params paramsMap, variables paramsMap) template.FuncMap { funcMap := getContainerServiceFuncMap(config) diff --git a/pkg/agent/nodecustomdata_render_test.go b/pkg/agent/nodecustomdata_render_test.go new file mode 100644 index 00000000000..d96cd1f197c --- /dev/null +++ b/pkg/agent/nodecustomdata_render_test.go @@ -0,0 +1,80 @@ +package agent + +import ( + "strings" + "testing" + + "github.com/Azure/agentbaker/pkg/agent/datamodel" + "github.com/stretchr/testify/require" +) + +func TestRenderLinuxNodeCustomDataTemplateUsesBakerPlatformFunctions(t *testing.T) { + template := []byte(`#cloud-config +write_files: +{{if IsACL}} +- path: /acl +{{else if IsAzlOSGuard}} +- path: /azlosguard +{{else if IsMariner}} +- path: /mariner +{{else if IsFlatcar}} +- path: /flatcar +{{else}} +- path: /ubuntu +{{end}} +`) + tests := []struct { + name string + distro datamodel.Distro + expected string + }{ + {name: "Ubuntu", distro: datamodel.AKSUbuntuContainerd2204Gen2, expected: "/ubuntu"}, + {name: "Mariner", distro: datamodel.AKSAzureLinuxV3Gen2, expected: "/mariner"}, + {name: "ACL", distro: datamodel.AKSACLGen2TL, expected: "/acl"}, + {name: "OS Guard", distro: datamodel.AKSAzureLinuxV3OSGuardGen2FIPSTL, expected: "/azlosguard"}, + {name: "Flatcar", distro: datamodel.AKSFlatcarGen2, expected: "/flatcar"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + rendered, err := RenderLinuxNodeCustomDataTemplate( + template, + newNodeCustomDataRenderConfig(test.distro), + ) + + require.NoError(t, err) + require.Contains(t, rendered, "- path: "+test.expected) + require.False(t, strings.Contains(rendered, "{{")) + }) + } +} + +func newNodeCustomDataRenderConfig(distro datamodel.Distro) *datamodel.NodeBootstrappingConfiguration { + profile := &datamodel.AgentPoolProfile{ + Name: "hotfix-render-test", + OSType: datamodel.Linux, + Distro: distro, + } + return &datamodel.NodeBootstrappingConfiguration{ + ContainerService: &datamodel.ContainerService{ + Location: "eastus", + Properties: &datamodel.Properties{ + OrchestratorProfile: &datamodel.OrchestratorProfile{ + OrchestratorVersion: "1.29.0", + OrchestratorType: datamodel.Kubernetes, + KubernetesConfig: &datamodel.KubernetesConfig{ + ContainerRuntimeConfig: map[string]string{}, + }, + }, + HostedMasterProfile: &datamodel.HostedMasterProfile{ + FQDN: "hotfix-render.invalid", + }, + AgentPoolProfiles: []*datamodel.AgentPoolProfile{profile}, + }, + }, + AgentPoolProfile: profile, + CloudSpecConfig: datamodel.AzurePublicCloudSpecForTest, + K8sComponents: &datamodel.K8sComponents{}, + KubeletConfig: map[string]string{}, + } +} From 1f9bfdcdd09e8c1c40a3ab71409b4c05f3a8065e Mon Sep 17 00:00:00 2001 From: Devin Wong Date: Thu, 3 Sep 2026 13:16:09 -0700 Subject: [PATCH 02/26] refactor: reuse ANC nodecustomdata application Share parsing, decoding, and transactional file application with the existing nodecustomdata path while keeping embedded distro selection independent of scripts_version. Preserve active generated payloads and pointers when no new hotfix is requested. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- aks-node-controller/README.md | 8 +- aks-node-controller/app.go | 5 +- aks-node-controller/app_test.go | 13 +- aks-node-controller/embeddednodecustomdata.go | 127 ++++++ .../embeddednodecustomdata_test.go | 387 ++++++++++++++++ aks-node-controller/nodecustomdata.go | 339 ++++++++++++-- aks-node-controller/nodecustomdata_test.go | 33 ++ aks-node-controller/scripthotfix/applier.go | 422 ------------------ .../scripthotfix/applier_test.go | 301 ------------- hotfix/hotfix_generate.py | 79 ++-- hotfix/hotfix_generate_test.py | 74 ++- 11 files changed, 968 insertions(+), 820 deletions(-) create mode 100644 aks-node-controller/embeddednodecustomdata.go create mode 100644 aks-node-controller/embeddednodecustomdata_test.go delete mode 100644 aks-node-controller/scripthotfix/applier.go delete mode 100644 aks-node-controller/scripthotfix/applier_test.go diff --git a/aks-node-controller/README.md b/aks-node-controller/README.md index e262f146631..16713ca5c7e 100644 --- a/aks-node-controller/README.md +++ b/aks-node-controller/README.md @@ -159,14 +159,18 @@ rendered nodecustomdata matching the local platform and atomically applies its Application is fail-open so the existing VHD scripts remain usable if validation or replacement fails. -The ANC-owned `scripthotfix` package distinguishes these embedded script hotfixes -from updates to the ANC binary itself. The generated files live under +The embedded nodecustomdata coordinator distinguishes these script hotfixes from +updates to the ANC binary itself. The generated files live under `aks-node-controller/scripthotfix/generated/` as `rendered_nodecustomdata_.yml`. The generator selects only changed hotfixable entries from `nodecustomdata.yml`, then renders Ubuntu, Mariner/Azure Linux, ACL, OS Guard, and Flatcar variants through AgentBaker's production Go-template functions. +When a PR has no new script hotfix, generation leaves the existing rendered +payload unchanged. The active ANC version pointer is likewise retained until it +is retired explicitly. + Embedded payloads are replace-only: ANC skips an entry when its runtime destination does not already exist. File presence preserves non-platform template gates such as custom-image exclusions. New-file hotfixes are not diff --git a/aks-node-controller/app.go b/aks-node-controller/app.go index 4893b3807b9..d92dc845f27 100644 --- a/aks-node-controller/app.go +++ b/aks-node-controller/app.go @@ -22,7 +22,6 @@ import ( "github.com/Azure/agentbaker/aks-node-controller/parser" "github.com/Azure/agentbaker/aks-node-controller/pkg/gpu" "github.com/Azure/agentbaker/aks-node-controller/pkg/nodeconfigutils" - "github.com/Azure/agentbaker/aks-node-controller/scripthotfix" "github.com/fsnotify/fsnotify" "github.com/urfave/cli/v3" ) @@ -73,7 +72,7 @@ type App struct { // is queried. fetchAttestedToken func(ctx context.Context) (string, error) // applyEmbeddedHotfix overrides embedded script application for tests. - applyEmbeddedHotfix func(string) (scripthotfix.Result, error) + applyEmbeddedHotfix func(string) (nodeCustomDataApplyResult, error) // grpcDialContext overrides how the gRPC LPS client dials, letting tests point the client at // an in-process (bufconn) server. When nil, the real TLS dial to the apiserver front is used. grpcDialContext func(ctx context.Context, target string) (net.Conn, error) @@ -690,7 +689,7 @@ func (a *App) Provision(ctx context.Context, flags ProvisionFlags) (*ProvisionRe func (a *App) applyEmbeddedHotfixPayload() { applyEmbeddedHotfix := a.applyEmbeddedHotfix if applyEmbeddedHotfix == nil { - applyEmbeddedHotfix = scripthotfix.ApplyEmbedded + applyEmbeddedHotfix = applyEmbeddedNodeCustomData } result, err := applyEmbeddedHotfix(a.osReleasePath) if err != nil { diff --git a/aks-node-controller/app_test.go b/aks-node-controller/app_test.go index 114e89c5005..1e162f92474 100644 --- a/aks-node-controller/app_test.go +++ b/aks-node-controller/app_test.go @@ -15,7 +15,6 @@ import ( "time" "github.com/Azure/agentbaker/aks-node-controller/helpers" - "github.com/Azure/agentbaker/aks-node-controller/scripthotfix" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -235,9 +234,9 @@ func TestApp_Provision(t *testing.T) { t.Run("embedded hotfix runs before command construction and execution", func(t *testing.T) { tt := NewTestApp(t, TestAppConfig{}) applied := false - tt.App.applyEmbeddedHotfix = func(string) (scripthotfix.Result, error) { + tt.App.applyEmbeddedHotfix = func(string) (nodeCustomDataApplyResult, error) { applied = true - return scripthotfix.Result{Applied: 1}, nil + return nodeCustomDataApplyResult{Applied: 1}, nil } _, err := tt.App.runProvision( @@ -259,8 +258,8 @@ func TestApp_Provision(t *testing.T) { return nil }, }) - tt.App.applyEmbeddedHotfix = func(string) (scripthotfix.Result, error) { - return scripthotfix.Result{}, errors.New("rendered nodecustomdata validation failed") + tt.App.applyEmbeddedHotfix = func(string) (nodeCustomDataApplyResult, error) { + return nodeCustomDataApplyResult{}, errors.New("rendered nodecustomdata validation failed") } _, err := tt.App.runProvision( @@ -281,9 +280,9 @@ func TestApp_Provision(t *testing.T) { t.Run("dry-run does not apply embedded hotfix payload", func(t *testing.T) { tt := NewTestApp(t, TestAppConfig{}) applied := false - tt.App.applyEmbeddedHotfix = func(string) (scripthotfix.Result, error) { + tt.App.applyEmbeddedHotfix = func(string) (nodeCustomDataApplyResult, error) { applied = true - return scripthotfix.Result{}, nil + return nodeCustomDataApplyResult{}, nil } _, err := tt.App.runProvision( diff --git a/aks-node-controller/embeddednodecustomdata.go b/aks-node-controller/embeddednodecustomdata.go new file mode 100644 index 00000000000..fe2aa7479a3 --- /dev/null +++ b/aks-node-controller/embeddednodecustomdata.go @@ -0,0 +1,127 @@ +package main + +import ( + "embed" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" +) + +const defaultOSReleasePath = "/etc/os-release" + +type nodeCustomDataPlatform string + +const ( + nodeCustomDataPlatformUbuntu nodeCustomDataPlatform = "ubuntu" + nodeCustomDataPlatformMariner nodeCustomDataPlatform = "mariner" + nodeCustomDataPlatformACL nodeCustomDataPlatform = "acl" + nodeCustomDataPlatformAzlOSGuard nodeCustomDataPlatform = "azlosguard" + nodeCustomDataPlatformFlatcar nodeCustomDataPlatform = "flatcar" +) + +//go:embed scripthotfix/generated +var embeddedGeneratedNodeCustomData embed.FS + +var generatedNodeCustomData fs.FS = embeddedGeneratedNodeCustomData + +func applyEmbeddedNodeCustomData(osReleasePath string) (nodeCustomDataApplyResult, error) { + active, err := fs.ReadFile(generatedNodeCustomData, "scripthotfix/generated/active") + if err != nil { + return nodeCustomDataApplyResult{}, fmt.Errorf("read embedded hotfix state: %w", err) + } + if strings.TrimSpace(string(active)) != "true" { + return nodeCustomDataApplyResult{}, nil + } + if osReleasePath == "" { + osReleasePath = defaultOSReleasePath + } + platform, err := classifyNodeCustomDataPlatform(osReleasePath) + if err != nil { + return nodeCustomDataApplyResult{}, err + } + return applyEmbeddedNodeCustomDataFS(generatedNodeCustomData, platform) +} + +func classifyNodeCustomDataPlatform(osReleasePath string) (nodeCustomDataPlatform, error) { + data, err := os.ReadFile(osReleasePath) + if err != nil { + return "", fmt.Errorf("read OS release %s: %w", osReleasePath, err) + } + values := parseNodeCustomDataOSRelease(data) + id := strings.ToLower(values["ID"]) + variant := strings.ToLower(values["VARIANT_ID"]) + + switch { + case variant == "osguard": + return nodeCustomDataPlatformAzlOSGuard, nil + case variant == "azurecontainerlinux", id == "azurecontainerlinux": + return nodeCustomDataPlatformACL, nil + case id == "ubuntu": + return nodeCustomDataPlatformUbuntu, nil + case id == "flatcar": + return nodeCustomDataPlatformFlatcar, nil + case id == "mariner", id == "azurelinux": + return nodeCustomDataPlatformMariner, nil + case id == "": + return "", fmt.Errorf("ID is missing from %s", osReleasePath) + default: + return "", fmt.Errorf("unsupported OS ID %q in %s", id, osReleasePath) + } +} + +func parseNodeCustomDataOSRelease(data []byte) map[string]string { + values := make(map[string]string) + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + key, value, found := strings.Cut(line, "=") + if !found { + continue + } + values[strings.TrimSpace(key)] = strings.Trim(strings.TrimSpace(value), `"'`) + } + return values +} + +func applyEmbeddedNodeCustomDataFS( + payloadFS fs.FS, + platform nodeCustomDataPlatform, +) (nodeCustomDataApplyResult, error) { + if !isConcreteNodeCustomDataPlatform(platform) { + return nodeCustomDataApplyResult{}, fmt.Errorf("unsupported concrete platform %q", platform) + } + renderedPath := filepath.ToSlash(filepath.Join( + "scripthotfix", + "generated", + fmt.Sprintf("rendered_nodecustomdata_%s.yml", platform), + )) + data, err := fs.ReadFile(payloadFS, renderedPath) + if err != nil { + return nodeCustomDataApplyResult{}, fmt.Errorf("read embedded nodecustomdata %s: %w", renderedPath, err) + } + return applyNodeCustomDataPayload(data, nodeCustomDataApplyOptions{ + source: renderedPath, + strict: true, + replaceOnly: true, + requirePermissions: true, + rejectUnsafePaths: true, + rejectEmptyContent: true, + }) +} + +func isConcreteNodeCustomDataPlatform(platform nodeCustomDataPlatform) bool { + switch platform { + case nodeCustomDataPlatformUbuntu, + nodeCustomDataPlatformMariner, + nodeCustomDataPlatformACL, + nodeCustomDataPlatformAzlOSGuard, + nodeCustomDataPlatformFlatcar: + return true + default: + return false + } +} diff --git a/aks-node-controller/embeddednodecustomdata_test.go b/aks-node-controller/embeddednodecustomdata_test.go new file mode 100644 index 00000000000..c5263983092 --- /dev/null +++ b/aks-node-controller/embeddednodecustomdata_test.go @@ -0,0 +1,387 @@ +package main + +import ( + "bytes" + "compress/gzip" + "encoding/base64" + "errors" + "os" + "path/filepath" + "runtime" + "testing" + "testing/fstest" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +func TestClassifyNodeCustomDataPlatform(t *testing.T) { + tests := []struct { + name string + release string + expected nodeCustomDataPlatform + }{ + {name: "Ubuntu", release: "ID=ubuntu\n", expected: nodeCustomDataPlatformUbuntu}, + {name: "Mariner", release: "ID=mariner\n", expected: nodeCustomDataPlatformMariner}, + {name: "Azure Linux", release: "ID=azurelinux\n", expected: nodeCustomDataPlatformMariner}, + { + name: "OS Guard variant wins over Azure Linux ID", + release: "ID=azurelinux\nVARIANT_ID=osguard\n", + expected: nodeCustomDataPlatformAzlOSGuard, + }, + { + name: "ACL variant wins over Azure Linux ID", + release: "ID=azurelinux\nVARIANT_ID=azurecontainerlinux\n", + expected: nodeCustomDataPlatformACL, + }, + {name: "ACL dedicated ID", release: "ID=azurecontainerlinux\n", expected: nodeCustomDataPlatformACL}, + {name: "Flatcar", release: "ID=flatcar\n", expected: nodeCustomDataPlatformFlatcar}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + releasePath := filepath.Join(t.TempDir(), "os-release") + require.NoError(t, os.WriteFile(releasePath, []byte(test.release), 0o600)) + + actual, err := classifyNodeCustomDataPlatform(releasePath) + + require.NoError(t, err) + assert.Equal(t, test.expected, actual) + }) + } + + t.Run("unsupported ID fails explicitly", func(t *testing.T) { + releasePath := filepath.Join(t.TempDir(), "os-release") + require.NoError(t, os.WriteFile(releasePath, []byte("ID=other\n"), 0o600)) + + _, err := classifyNodeCustomDataPlatform(releasePath) + + require.ErrorContains(t, err, "unsupported OS ID") + }) +} + +func TestApplyEmbeddedNodeCustomDataInactiveDoesNotReadOSRelease(t *testing.T) { + original := generatedNodeCustomData + generatedNodeCustomData = fstest.MapFS{ + "scripthotfix/generated/active": &fstest.MapFile{Data: []byte("false\n")}, + } + t.Cleanup(func() { + generatedNodeCustomData = original + }) + + result, err := applyEmbeddedNodeCustomData(filepath.Join(t.TempDir(), "missing-os-release")) + + require.NoError(t, err) + assert.Equal(t, nodeCustomDataApplyResult{}, result) +} + +func TestApplyEmbeddedNodeCustomDataSelectsPlatformPayload(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Windows rename cannot atomically replace an existing destination") + } + + directory := t.TempDir() + destination := filepath.Join(directory, "provision.sh") + require.NoError(t, os.WriteFile(destination, []byte("old"), 0o600)) + releasePath := filepath.Join(directory, "os-release") + require.NoError(t, os.WriteFile(releasePath, []byte("ID=ubuntu\n"), 0o600)) + payload := []byte("#!/bin/sh\necho fixed\n") + + original := generatedNodeCustomData + generatedNodeCustomData = fstest.MapFS{ + "scripthotfix/generated/active": &fstest.MapFile{Data: []byte("true\n")}, + embeddedRenderedPath(nodeCustomDataPlatformUbuntu): &fstest.MapFile{Data: marshalNodeCustomData(t, []nodeCustomDataWriteFile{{ + Path: destination, + Permissions: "0744", + Encoding: encodingBase64, + Owner: "root", + Content: base64.StdEncoding.EncodeToString(payload), + }})}, + } + t.Cleanup(func() { + generatedNodeCustomData = original + }) + + result, err := applyEmbeddedNodeCustomData(releasePath) + + require.NoError(t, err) + assert.Equal(t, nodeCustomDataApplyResult{Applied: 1}, result) + actual, err := os.ReadFile(destination) + require.NoError(t, err) + assert.Equal(t, payload, actual) +} + +func TestApplyEmbeddedNodeCustomDataIsReplaceOnlyAndIdempotent(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Windows rename cannot atomically replace an existing destination") + } + + directory := t.TempDir() + destination := filepath.Join(directory, "provision.sh") + missing := filepath.Join(directory, "missing.sh") + require.NoError(t, os.WriteFile(destination, []byte("old"), 0o600)) + files := embeddedRenderedFS(t, nodeCustomDataPlatformUbuntu, []nodeCustomDataWriteFile{ + { + Path: destination, + Permissions: "0744", + Owner: "root", + Content: "hotfix", + }, + { + Path: missing, + Permissions: "0744", + Owner: "root", + Content: "not-created", + }, + }) + + first, err := applyEmbeddedNodeCustomDataFS(files, nodeCustomDataPlatformUbuntu) + + require.NoError(t, err) + assert.Equal(t, nodeCustomDataApplyResult{Applied: 1, Skipped: 1}, first) + actual, err := os.ReadFile(destination) + require.NoError(t, err) + assert.Equal(t, []byte("hotfix"), actual) + info, err := os.Stat(destination) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o744), info.Mode().Perm()) + _, statErr := os.Stat(missing) + assert.True(t, os.IsNotExist(statErr)) + + second, err := applyEmbeddedNodeCustomDataFS(files, nodeCustomDataPlatformUbuntu) + require.NoError(t, err) + assert.Equal(t, nodeCustomDataApplyResult{Skipped: 2}, second) +} + +func TestEmbeddedNodeCustomDataStrictValidation(t *testing.T) { + validDestination := filepath.Join(t.TempDir(), "provision.sh") + valid := nodeCustomDataWriteFile{ + Path: validDestination, + Permissions: "0744", + Owner: "root", + Content: "hotfix", + } + + tests := []struct { + name string + files []nodeCustomDataWriteFile + expectedErr string + }{ + { + name: "unsafe destination", + files: []nodeCustomDataWriteFile{{ + Path: "../provision.sh", + Permissions: "0744", + Owner: "root", + Content: "hotfix", + }}, + expectedErr: "unsafe destination", + }, + { + name: "destination with embedded backslash", + files: []nodeCustomDataWriteFile{{ + Path: `/opt/provision\script.sh`, + Permissions: "0744", + Owner: "root", + Content: "hotfix", + }}, + expectedErr: "backslashes are not allowed", + }, + { + name: "absent mode", + files: []nodeCustomDataWriteFile{{ + Path: validDestination, + Owner: "root", + Content: "hotfix", + }}, + expectedErr: "invalid mode", + }, + { + name: "invalid mode", + files: []nodeCustomDataWriteFile{{ + Path: validDestination, + Permissions: "0999", + Owner: "root", + Content: "hotfix", + }}, + expectedErr: "invalid mode", + }, + { + name: "unsupported owner", + files: []nodeCustomDataWriteFile{{ + Path: validDestination, + Permissions: "0744", + Owner: "nobody", + Content: "hotfix", + }}, + expectedErr: "unsupported owner", + }, + { + name: "unsupported encoding", + files: []nodeCustomDataWriteFile{{ + Path: validDestination, + Permissions: "0744", + Owner: "root", + Encoding: "rot13", + Content: "hotfix", + }}, + expectedErr: "unsupported encoding", + }, + { + name: "empty decoded content", + files: []nodeCustomDataWriteFile{{ + Path: validDestination, + Permissions: "0744", + Owner: "root", + Encoding: encodingBase64, + Content: "", + }}, + expectedErr: "is empty", + }, + { + name: "duplicate destination", + files: []nodeCustomDataWriteFile{valid, valid}, + expectedErr: "duplicate destination", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := applyEmbeddedNodeCustomDataFS( + embeddedRenderedFS(t, nodeCustomDataPlatformUbuntu, test.files), + nodeCustomDataPlatformUbuntu, + ) + require.ErrorContains(t, err, test.expectedErr) + }) + } + + t.Run("unknown YAML field", func(t *testing.T) { + files := fstest.MapFS{ + embeddedRenderedPath(nodeCustomDataPlatformUbuntu): &fstest.MapFile{ + Data: []byte("write_files: []\nunknown: true\n"), + }, + } + _, err := applyEmbeddedNodeCustomDataFS(files, nodeCustomDataPlatformUbuntu) + require.ErrorContains(t, err, "field unknown not found") + }) + + t.Run("trailing YAML document", func(t *testing.T) { + files := fstest.MapFS{ + embeddedRenderedPath(nodeCustomDataPlatformUbuntu): &fstest.MapFile{ + Data: []byte("write_files: []\n---\nwrite_files: []\n"), + }, + } + _, err := applyEmbeddedNodeCustomDataFS(files, nodeCustomDataPlatformUbuntu) + require.ErrorContains(t, err, "trailing content") + }) +} + +func TestNodeCustomDataSharedDecoderHandlesGzip(t *testing.T) { + var compressed bytes.Buffer + writer := gzip.NewWriter(&compressed) + _, err := writer.Write([]byte("rendered payload")) + require.NoError(t, err) + require.NoError(t, writer.Close()) + + decoded, err := decodeNodeCustomDataWriteFileContent(nodeCustomDataWriteFile{ + Encoding: encodingGZIP, + Content: compressed.String(), + }) + + require.NoError(t, err) + assert.Equal(t, []byte("rendered payload"), decoded) +} + +func TestEmbeddedNodeCustomDataStagesAllBeforeCommit(t *testing.T) { + directory := t.TempDir() + firstDestination := filepath.Join(directory, "first.sh") + require.NoError(t, os.WriteFile(firstDestination, []byte("original"), 0o700)) + files := embeddedRenderedFS(t, nodeCustomDataPlatformUbuntu, []nodeCustomDataWriteFile{ + { + Path: firstDestination, + Permissions: "0744", + Owner: "root", + Content: "first hotfix", + }, + { + Path: directory, + Permissions: "0744", + Owner: "root", + Content: "second hotfix", + }, + }) + + _, err := applyEmbeddedNodeCustomDataFS(files, nodeCustomDataPlatformUbuntu) + + require.ErrorContains(t, err, "read destination") + actual, readErr := os.ReadFile(firstDestination) + require.NoError(t, readErr) + assert.Equal(t, []byte("original"), actual) + staged, globErr := filepath.Glob(filepath.Join(directory, ".aks-node-controller-nodecustomdata-*")) + require.NoError(t, globErr) + assert.Empty(t, staged) +} + +func TestCommitStagedNodeCustomDataRollsBackEarlierReplacement(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Windows rename cannot atomically replace an existing destination") + } + directory := t.TempDir() + firstDestination := filepath.Join(directory, "first.sh") + secondDestination := filepath.Join(directory, "second.sh") + require.NoError(t, os.WriteFile(firstDestination, []byte("first original"), 0o700)) + require.NoError(t, os.WriteFile(secondDestination, []byte("second original"), 0o711)) + first, changed, err := stageNodeCustomDataEntry( + nodeCustomDataEntry{destination: firstDestination, content: []byte("first hotfix"), mode: 0o744}, + true, + ) + require.NoError(t, err) + require.True(t, changed) + second, changed, err := stageNodeCustomDataEntry( + nodeCustomDataEntry{destination: secondDestination, content: []byte("second hotfix"), mode: 0o755}, + true, + ) + require.NoError(t, err) + require.True(t, changed) + + err = commitStagedNodeCustomDataWithRename( + []*stagedNodeCustomDataEntry{&first, &second}, + func(source string, destination string) error { + if source == second.stagedPath { + return errors.New("injected rename failure") + } + return os.Rename(source, destination) + }, + ) + + require.ErrorContains(t, err, "injected rename failure") + firstActual, readErr := os.ReadFile(firstDestination) + require.NoError(t, readErr) + assert.Equal(t, []byte("first original"), firstActual) + secondActual, readErr := os.ReadFile(secondDestination) + require.NoError(t, readErr) + assert.Equal(t, []byte("second original"), secondActual) +} + +func embeddedRenderedFS( + t *testing.T, + platform nodeCustomDataPlatform, + files []nodeCustomDataWriteFile, +) fstest.MapFS { + t.Helper() + return fstest.MapFS{ + embeddedRenderedPath(platform): &fstest.MapFile{Data: marshalNodeCustomData(t, files)}, + } +} + +func marshalNodeCustomData(t *testing.T, files []nodeCustomDataWriteFile) []byte { + t.Helper() + data, err := yaml.Marshal(nodeCustomData{WriteFiles: files}) + require.NoError(t, err) + return data +} + +func embeddedRenderedPath(platform nodeCustomDataPlatform) string { + return "scripthotfix/generated/rendered_nodecustomdata_" + string(platform) + ".yml" +} diff --git a/aks-node-controller/nodecustomdata.go b/aks-node-controller/nodecustomdata.go index 0d178d81e59..de4db972e17 100644 --- a/aks-node-controller/nodecustomdata.go +++ b/aks-node-controller/nodecustomdata.go @@ -4,11 +4,14 @@ import ( "bytes" "compress/gzip" "encoding/base64" + "errors" "fmt" "io" "os" + "path" "path/filepath" "strconv" + "strings" "gopkg.in/yaml.v3" ) @@ -31,60 +34,169 @@ type nodeCustomData struct { WriteFiles []nodeCustomDataWriteFile `yaml:"write_files"` } -func applyNodeCustomData(path string) error { - data, err := os.ReadFile(path) +type nodeCustomDataApplyOptions struct { + source string + strict bool + replaceOnly bool + requirePermissions bool + rejectUnsafePaths bool + rejectEmptyContent bool +} + +type nodeCustomDataApplyResult struct { + Applied int + Skipped int +} + +type nodeCustomDataEntry struct { + destination string + mode os.FileMode + content []byte +} + +type stagedNodeCustomDataEntry struct { + destination string + stagedPath string + backupPath string + originalExist bool + preserveBackup bool +} + +func applyNodeCustomData(nodeCustomDataPath string) error { + data, err := os.ReadFile(nodeCustomDataPath) if err != nil { if os.IsNotExist(err) { return nil } - return fmt.Errorf("read nodecustomdata %s: %w", path, err) + return fmt.Errorf("read nodecustomdata %s: %w", nodeCustomDataPath, err) } - var customData nodeCustomData - if err := yaml.Unmarshal(data, &customData); err != nil { - return fmt.Errorf("unmarshal nodecustomdata %s: %w", path, err) + if _, err := applyNodeCustomDataPayload(data, nodeCustomDataApplyOptions{source: nodeCustomDataPath}); err != nil { + return fmt.Errorf("apply nodecustomdata %s: %w", nodeCustomDataPath, err) + } + return nil +} + +func applyNodeCustomDataPayload(data []byte, options nodeCustomDataApplyOptions) (nodeCustomDataApplyResult, error) { + customData, err := parseNodeCustomData(data, options) + if err != nil { + return nodeCustomDataApplyResult{}, err + } + + entries, err := validateNodeCustomData(customData, options) + if err != nil { + return nodeCustomDataApplyResult{}, err } - for _, file := range customData.WriteFiles { - if err := applyNodeCustomDataWriteFile(file); err != nil { - return fmt.Errorf("apply nodecustomdata write file %s: %w", file.Path, err) + result := nodeCustomDataApplyResult{} + var staged []*stagedNodeCustomDataEntry + for _, entry := range entries { + pending, changed, err := stageNodeCustomDataEntry(entry, options.replaceOnly) + if err != nil { + cleanupStagedNodeCustomData(staged) + return result, fmt.Errorf("stage nodecustomdata destination %s: %w", entry.destination, err) } + if !changed { + result.Skipped++ + continue + } + staged = append(staged, &pending) } - return nil + if err := commitStagedNodeCustomData(staged); err != nil { + return nodeCustomDataApplyResult{}, err + } + result.Applied = len(staged) + return result, nil } -func applyNodeCustomDataWriteFile(file nodeCustomDataWriteFile) error { - if file.Path == "" { - return fmt.Errorf("path is required") +func parseNodeCustomData(data []byte, options nodeCustomDataApplyOptions) (nodeCustomData, error) { + var customData nodeCustomData + if !options.strict { + if err := yaml.Unmarshal(data, &customData); err != nil { + return nodeCustomData{}, fmt.Errorf("unmarshal nodecustomdata %s: %w", options.source, err) + } + return customData, nil } - if file.Owner != "" && file.Owner != "root" { - return fmt.Errorf("unsupported owner %q", file.Owner) + + decoder := yaml.NewDecoder(bytes.NewReader(data)) + decoder.KnownFields(true) + if err := decoder.Decode(&customData); err != nil { + return nodeCustomData{}, fmt.Errorf("decode nodecustomdata %s: %w", options.source, err) + } + var trailing any + if err := decoder.Decode(&trailing); err != io.EOF { + return nodeCustomData{}, fmt.Errorf("nodecustomdata %s has trailing content", options.source) } + return customData, nil +} - mode := os.FileMode(0o644) - if file.Permissions != "" { - parsedMode, err := strconv.ParseUint(file.Permissions, 8, 32) +func validateNodeCustomData(customData nodeCustomData, options nodeCustomDataApplyOptions) ([]nodeCustomDataEntry, error) { + entries := make([]nodeCustomDataEntry, 0, len(customData.WriteFiles)) + destinations := make(map[string]struct{}, len(customData.WriteFiles)) + for index, file := range customData.WriteFiles { + entry, err := validateNodeCustomDataWriteFile(file, options) if err != nil { - return fmt.Errorf("parse permissions: %w", err) + return nil, fmt.Errorf("validate write_files entry %d: %w", index, err) + } + if options.strict { + if _, exists := destinations[entry.destination]; exists { + return nil, fmt.Errorf("duplicate destination %s", entry.destination) + } + destinations[entry.destination] = struct{}{} } - mode = os.FileMode(parsedMode) + entries = append(entries, entry) } + return entries, nil +} - contents, err := decodeNodeCustomDataWriteFileContent(file) - if err != nil { - return err +func validateNodeCustomDataWriteFile(file nodeCustomDataWriteFile, options nodeCustomDataApplyOptions) (nodeCustomDataEntry, error) { + if file.Path == "" { + return nodeCustomDataEntry{}, fmt.Errorf("path is required") } - - if err := os.MkdirAll(filepath.Dir(file.Path), 0o755); err != nil { - return fmt.Errorf("create parent directory: %w", err) + if options.rejectUnsafePaths { + if (!strings.HasPrefix(file.Path, "/") && !filepath.IsAbs(file.Path)) || + (strings.HasPrefix(file.Path, "/") && path.Clean(file.Path) != file.Path) || + (!strings.HasPrefix(file.Path, "/") && filepath.Clean(file.Path) != file.Path) { + return nodeCustomDataEntry{}, fmt.Errorf("unsafe destination %q", file.Path) + } + if strings.HasPrefix(file.Path, "/") && strings.Contains(file.Path, `\`) { + return nodeCustomDataEntry{}, fmt.Errorf("unsafe destination %q: backslashes are not allowed", file.Path) + } + } + if file.Owner != "" && file.Owner != "root" { + return nodeCustomDataEntry{}, fmt.Errorf("unsupported owner %q", file.Owner) } - if err := os.WriteFile(file.Path, contents, mode); err != nil { - return fmt.Errorf("write file: %w", err) + mode, err := parseNodeCustomDataMode(file.Permissions, options.requirePermissions) + if err != nil { + return nodeCustomDataEntry{}, err + } + content, err := decodeNodeCustomDataWriteFileContent(file) + if err != nil { + return nodeCustomDataEntry{}, err + } + if options.rejectEmptyContent && len(content) == 0 { + return nodeCustomDataEntry{}, fmt.Errorf("content for %s is empty", file.Path) } + return nodeCustomDataEntry{destination: file.Path, mode: mode, content: content}, nil +} - return nil +func parseNodeCustomDataMode(value string, required bool) (os.FileMode, error) { + if value == "" && !required { + return 0o644, nil + } + parsed, err := strconv.ParseUint(value, 8, 32) + if err != nil { + if required { + return 0, fmt.Errorf("invalid mode %q", value) + } + return 0, fmt.Errorf("parse permissions: %w", err) + } + if required && (parsed == 0 || parsed > 0o777) { + return 0, fmt.Errorf("invalid mode %q", value) + } + return os.FileMode(parsed), nil } func decodeNodeCustomDataWriteFileContent(file nodeCustomDataWriteFile) ([]byte, error) { @@ -94,22 +206,185 @@ func decodeNodeCustomDataWriteFileContent(file nodeCustomDataWriteFile) ([]byte, case encodingGZIP: reader, err := gzip.NewReader(bytes.NewReader([]byte(file.Content))) if err != nil { - return nil, fmt.Errorf("create gzip reader: %w", err) + return nil, fmt.Errorf("create gzip reader for %s: %w", file.Path, err) } defer reader.Close() decoded, err := io.ReadAll(reader) if err != nil { - return nil, fmt.Errorf("read gzip content: %w", err) + return nil, fmt.Errorf("read gzip content for %s: %w", file.Path, err) } return decoded, nil case encodingBase64: decoded, err := base64.StdEncoding.DecodeString(file.Content) if err != nil { - return nil, fmt.Errorf("decode base64 content: %w", err) + return nil, fmt.Errorf("decode base64 content for %s: %w", file.Path, err) } return decoded, nil default: return nil, fmt.Errorf("unsupported encoding %q", file.Encoding) } } + +func stageNodeCustomDataEntry(entry nodeCustomDataEntry, replaceOnly bool) (stagedNodeCustomDataEntry, bool, error) { + current, err := os.ReadFile(entry.destination) + originalExists := err == nil + var originalMode os.FileMode + switch { + case err == nil: + info, statErr := os.Stat(entry.destination) + if statErr != nil { + return stagedNodeCustomDataEntry{}, false, fmt.Errorf("stat destination: %w", statErr) + } + originalMode = info.Mode().Perm() + if bytes.Equal(current, entry.content) && originalMode == entry.mode.Perm() { + return stagedNodeCustomDataEntry{}, false, nil + } + case os.IsNotExist(err): + if replaceOnly { + return stagedNodeCustomDataEntry{}, false, nil + } + default: + return stagedNodeCustomDataEntry{}, false, fmt.Errorf("read destination: %w", err) + } + + directory := filepath.Dir(entry.destination) + if originalExists { + info, statErr := os.Stat(directory) + if statErr != nil { + return stagedNodeCustomDataEntry{}, false, fmt.Errorf("stat destination directory %s: %w", directory, statErr) + } + if !info.IsDir() { + return stagedNodeCustomDataEntry{}, false, fmt.Errorf("destination parent %s is not a directory", directory) + } + } else if err := os.MkdirAll(directory, 0o755); err != nil { + return stagedNodeCustomDataEntry{}, false, fmt.Errorf("create parent directory: %w", err) + } + + stagedPath, err := writeNodeCustomDataTempFile(directory, ".aks-node-controller-nodecustomdata-stage-*", entry.content, entry.mode) + if err != nil { + return stagedNodeCustomDataEntry{}, false, err + } + staged := stagedNodeCustomDataEntry{ + destination: entry.destination, + stagedPath: stagedPath, + originalExist: originalExists, + } + if !originalExists { + return staged, true, nil + } + + backupPath, err := writeNodeCustomDataTempFile( + directory, + ".aks-node-controller-nodecustomdata-backup-*", + current, + originalMode, + ) + if err != nil { + _ = os.Remove(stagedPath) + return stagedNodeCustomDataEntry{}, false, fmt.Errorf("stage destination backup: %w", err) + } + staged.backupPath = backupPath + return staged, true, nil +} + +func writeNodeCustomDataTempFile(directory, pattern string, content []byte, mode os.FileMode) (string, error) { + temp, err := os.CreateTemp(directory, pattern) + if err != nil { + return "", fmt.Errorf("create temporary file: %w", err) + } + tempPath := temp.Name() + cleanup := func() { + _ = temp.Close() + _ = os.Remove(tempPath) + } + if _, err := temp.Write(content); err != nil { + cleanup() + return "", fmt.Errorf("write temporary file: %w", err) + } + if err := temp.Sync(); err != nil { + cleanup() + return "", fmt.Errorf("sync temporary file: %w", err) + } + if err := temp.Close(); err != nil { + cleanup() + return "", fmt.Errorf("close temporary file: %w", err) + } + if err := os.Chmod(tempPath, mode); err != nil { + cleanup() + return "", fmt.Errorf("chmod temporary file: %w", err) + } + return tempPath, nil +} + +func commitStagedNodeCustomData(staged []*stagedNodeCustomDataEntry) error { + return commitStagedNodeCustomDataWithRename(staged, os.Rename) +} + +func commitStagedNodeCustomDataWithRename( + staged []*stagedNodeCustomDataEntry, + rename func(string, string) error, +) error { + committed := 0 + defer cleanupStagedNodeCustomData(staged) + for index, entry := range staged { + if err := rename(entry.stagedPath, entry.destination); err != nil { + rollbackErr := rollbackStagedNodeCustomData(staged[:committed], rename) + if rollbackErr != nil { + return fmt.Errorf( + "commit nodecustomdata destination %s: %w; rollback failed: %w", + entry.destination, + err, + rollbackErr, + ) + } + return fmt.Errorf("commit nodecustomdata destination %s: %w", entry.destination, err) + } + staged[index].stagedPath = "" + committed++ + } + return nil +} + +func rollbackStagedNodeCustomData( + committed []*stagedNodeCustomDataEntry, + rename func(string, string) error, +) error { + var rollbackErrors []error + for index := len(committed) - 1; index >= 0; index-- { + entry := committed[index] + var err error + if entry.originalExist { + err = rename(entry.backupPath, entry.destination) + if err == nil { + entry.backupPath = "" + } + } else { + err = os.Remove(entry.destination) + } + if err != nil { + entry.preserveBackup = true + rollbackErrors = append( + rollbackErrors, + fmt.Errorf( + "restore %s from preserved backup %s: %w", + entry.destination, + entry.backupPath, + err, + ), + ) + } + } + return errors.Join(rollbackErrors...) +} + +func cleanupStagedNodeCustomData(staged []*stagedNodeCustomDataEntry) { + for _, entry := range staged { + if entry.stagedPath != "" { + _ = os.Remove(entry.stagedPath) + } + if entry.backupPath != "" && !entry.preserveBackup { + _ = os.Remove(entry.backupPath) + } + } +} diff --git a/aks-node-controller/nodecustomdata_test.go b/aks-node-controller/nodecustomdata_test.go index 46bdbffae01..65e6192a87b 100644 --- a/aks-node-controller/nodecustomdata_test.go +++ b/aks-node-controller/nodecustomdata_test.go @@ -8,6 +8,7 @@ import ( "fmt" "os" "path/filepath" + "runtime" "testing" "github.com/stretchr/testify/assert" @@ -53,6 +54,38 @@ write_files: assert.Equal(t, "gzip-content", string(gzipContent)) } +func TestApplyNodeCustomDataPreservesLegacyDefaultsAndCreatesParents(t *testing.T) { + tempDir := t.TempDir() + destination := filepath.Join(tempDir, "missing", "parent", "payload.txt") + renderedPath := filepath.Join(tempDir, "nodecustomdata.yml") + content := base64.StdEncoding.EncodeToString([]byte("base64-content")) + rendered := fmt.Sprintf(`write_files: +- path: %s + owner: root + encoding: base64 + content: %s +`, destination, content) + require.NoError(t, os.WriteFile(renderedPath, []byte(rendered), 0o600)) + + require.NoError(t, applyNodeCustomData(renderedPath)) + + actual, err := os.ReadFile(destination) + require.NoError(t, err) + assert.Equal(t, []byte("base64-content"), actual) + mode, err := parseNodeCustomDataMode("", false) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o644), mode) + if runtime.GOOS != "windows" { + info, statErr := os.Stat(destination) + require.NoError(t, statErr) + assert.Equal(t, os.FileMode(0o644), info.Mode().Perm()) + } +} + +func TestApplyNodeCustomDataMissingFileIsNoOp(t *testing.T) { + require.NoError(t, applyNodeCustomData(filepath.Join(t.TempDir(), "missing.yml"))) +} + // TestDownloadHotfixAppliesRenderedWriteFilesWhenScriptsVersionMatches verifies that // downloadHotfix applies the rendered nodecustomdata write_files when the hotfix config's // scripts_version targets the current ANC version's YYYYMM.DD base with a strictly higher patch. diff --git a/aks-node-controller/scripthotfix/applier.go b/aks-node-controller/scripthotfix/applier.go deleted file mode 100644 index ddfaae4e5fc..00000000000 --- a/aks-node-controller/scripthotfix/applier.go +++ /dev/null @@ -1,422 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. - -// Package scripthotfix applies rendered provisioning script hotfixes embedded -// in the aks-node-controller binary. -package scripthotfix - -import ( - "bytes" - "compress/gzip" - "embed" - "encoding/base64" - "errors" - "fmt" - "io" - "io/fs" - "os" - "path" - "path/filepath" - "strconv" - "strings" - - "gopkg.in/yaml.v3" -) - -const defaultOSReleasePath = "/etc/os-release" - -type Platform string - -const ( - PlatformUbuntu Platform = "ubuntu" - PlatformMariner Platform = "mariner" - PlatformACL Platform = "acl" - PlatformAzlOSGuard Platform = "azlosguard" - PlatformFlatcar Platform = "flatcar" -) - -//go:embed generated -var embeddedGeneratedFiles embed.FS - -var generatedFiles fs.FS = embeddedGeneratedFiles - -type nodeCustomData struct { - WriteFiles []writeFile `yaml:"write_files"` -} - -type writeFile struct { - Path string `yaml:"path"` - Permissions string `yaml:"permissions"` - Encoding string `yaml:"encoding,omitempty"` - Owner string `yaml:"owner"` - Content string `yaml:"content"` -} - -type payloadEntry struct { - destination string - mode os.FileMode - content []byte -} - -type Result struct { - Applied int - Skipped int -} - -type stagedEntry struct { - destination string - stagedPath string - backupPath string - originalExist bool - preserveBackup bool -} - -// ApplyEmbedded applies the rendered payload compiled into this ANC binary. -func ApplyEmbedded(osReleasePath string) (Result, error) { - active, err := fs.ReadFile(generatedFiles, "generated/active") - if err != nil { - return Result{}, fmt.Errorf("read embedded hotfix state: %w", err) - } - if strings.TrimSpace(string(active)) != "true" { - return Result{}, nil - } - if osReleasePath == "" { - osReleasePath = defaultOSReleasePath - } - platform, err := ClassifyPlatform(osReleasePath) - if err != nil { - return Result{}, err - } - return applyFS(generatedFiles, platform) -} - -// ClassifyPlatform maps /etc/os-release to rendered nodecustomdata variants. -func ClassifyPlatform(osReleasePath string) (Platform, error) { - data, err := os.ReadFile(osReleasePath) - if err != nil { - return "", fmt.Errorf("read OS release %s: %w", osReleasePath, err) - } - values := parseOSRelease(data) - id := strings.ToLower(values["ID"]) - variant := strings.ToLower(values["VARIANT_ID"]) - - switch { - case variant == "osguard": - return PlatformAzlOSGuard, nil - case variant == "azurecontainerlinux", id == "azurecontainerlinux": - return PlatformACL, nil - case id == "ubuntu": - return PlatformUbuntu, nil - case id == "flatcar": - return PlatformFlatcar, nil - case id == "mariner", id == "azurelinux": - return PlatformMariner, nil - case id == "": - return "", fmt.Errorf("ID is missing from %s", osReleasePath) - default: - return "", fmt.Errorf("unsupported OS ID %q in %s", id, osReleasePath) - } -} - -func parseOSRelease(data []byte) map[string]string { - values := make(map[string]string) - for _, line := range strings.Split(string(data), "\n") { - line = strings.TrimSpace(line) - if line == "" || strings.HasPrefix(line, "#") { - continue - } - key, value, found := strings.Cut(line, "=") - if !found { - continue - } - values[strings.TrimSpace(key)] = strings.Trim(strings.TrimSpace(value), `"'`) - } - return values -} - -func applyFS(payloadFS fs.FS, platform Platform) (Result, error) { - entries, err := loadAndValidate(payloadFS, platform) - if err != nil { - return Result{}, err - } - - result := Result{} - var staged []*stagedEntry - for _, entry := range entries { - pending, changed, err := stageEntry(entry.destination, entry.content, entry.mode) - if err != nil { - cleanupStaged(staged) - return result, fmt.Errorf("apply embedded hotfix to %s: %w", entry.destination, err) - } - if !changed { - result.Skipped++ - continue - } - staged = append(staged, &pending) - } - if err := commitStaged(staged); err != nil { - return Result{}, err - } - result.Applied = len(staged) - return result, nil -} - -func loadAndValidate(payloadFS fs.FS, platform Platform) ([]payloadEntry, error) { - if !isConcretePlatform(platform) { - return nil, fmt.Errorf("unsupported concrete platform %q", platform) - } - renderedPath := fmt.Sprintf( - "generated/rendered_nodecustomdata_%s.yml", - platform, - ) - data, err := fs.ReadFile(payloadFS, renderedPath) - if err != nil { - return nil, fmt.Errorf("read embedded nodecustomdata %s: %w", renderedPath, err) - } - - var customData nodeCustomData - decoder := yaml.NewDecoder(bytes.NewReader(data)) - decoder.KnownFields(true) - if err := decoder.Decode(&customData); err != nil { - return nil, fmt.Errorf("decode embedded nodecustomdata %s: %w", renderedPath, err) - } - var trailing any - if err := decoder.Decode(&trailing); err != io.EOF { - return nil, fmt.Errorf("embedded nodecustomdata %s has trailing content", renderedPath) - } - - entries := make([]payloadEntry, 0, len(customData.WriteFiles)) - destinations := make(map[string]struct{}, len(customData.WriteFiles)) - for index, file := range customData.WriteFiles { - entry, err := validateWriteFile(file) - if err != nil { - return nil, fmt.Errorf("validate embedded write_files entry %d: %w", index, err) - } - if _, exists := destinations[entry.destination]; exists { - return nil, fmt.Errorf("duplicate destination %s", entry.destination) - } - destinations[entry.destination] = struct{}{} - entries = append(entries, entry) - } - return entries, nil -} - -func validateWriteFile(file writeFile) (payloadEntry, error) { - if file.Path == "" || - (!strings.HasPrefix(file.Path, "/") && !filepath.IsAbs(file.Path)) || - (strings.HasPrefix(file.Path, "/") && path.Clean(file.Path) != file.Path) || - (!strings.HasPrefix(file.Path, "/") && filepath.Clean(file.Path) != file.Path) { - return payloadEntry{}, fmt.Errorf("unsafe destination %q", file.Path) - } - if strings.HasPrefix(file.Path, "/") && strings.Contains(file.Path, `\`) { - return payloadEntry{}, fmt.Errorf("unsafe destination %q: backslashes are not allowed", file.Path) - } - if file.Owner != "" && file.Owner != "root" { - return payloadEntry{}, fmt.Errorf("unsupported owner %q", file.Owner) - } - mode, err := parseMode(file.Permissions) - if err != nil { - return payloadEntry{}, err - } - content, err := decodeContent(file) - if err != nil { - return payloadEntry{}, err - } - if len(content) == 0 { - return payloadEntry{}, fmt.Errorf("content for %s is empty", file.Path) - } - return payloadEntry{ - destination: file.Path, - mode: mode, - content: content, - }, nil -} - -func parseMode(value string) (os.FileMode, error) { - parsed, err := strconv.ParseUint(value, 8, 32) - if err != nil || parsed == 0 || parsed > 0o777 { - return 0, fmt.Errorf("invalid mode %q", value) - } - return os.FileMode(parsed), nil -} - -func decodeContent(file writeFile) ([]byte, error) { - switch file.Encoding { - case "": - return []byte(file.Content), nil - case "base64": - decoded, err := base64.StdEncoding.DecodeString(file.Content) - if err != nil { - return nil, fmt.Errorf("decode base64 content for %s: %w", file.Path, err) - } - return decoded, nil - case "gzip": - reader, err := gzip.NewReader(bytes.NewReader([]byte(file.Content))) - if err != nil { - return nil, fmt.Errorf("create gzip reader for %s: %w", file.Path, err) - } - defer reader.Close() - decoded, err := io.ReadAll(reader) - if err != nil { - return nil, fmt.Errorf("read gzip content for %s: %w", file.Path, err) - } - return decoded, nil - default: - return nil, fmt.Errorf("unsupported encoding %q", file.Encoding) - } -} - -func isConcretePlatform(platform Platform) bool { - switch platform { - case PlatformUbuntu, PlatformMariner, PlatformACL, PlatformAzlOSGuard, PlatformFlatcar: - return true - default: - return false - } -} - -func stageEntry(destination string, payload []byte, mode os.FileMode) (stagedEntry, bool, error) { - current, err := os.ReadFile(destination) - var originalMode os.FileMode - switch { - case err == nil: - info, statErr := os.Stat(destination) - if statErr != nil { - return stagedEntry{}, false, fmt.Errorf("stat destination: %w", statErr) - } - originalMode = info.Mode().Perm() - if bytes.Equal(current, payload) && originalMode == mode.Perm() { - return stagedEntry{}, false, nil - } - case os.IsNotExist(err): - return stagedEntry{}, false, nil - default: - return stagedEntry{}, false, fmt.Errorf("read destination: %w", err) - } - - directory := filepath.Dir(destination) - info, err := os.Stat(directory) - if err != nil { - return stagedEntry{}, false, fmt.Errorf("stat destination directory %s: %w", directory, err) - } - if !info.IsDir() { - return stagedEntry{}, false, fmt.Errorf("destination parent %s is not a directory", directory) - } - - stagedPath, err := writeTempFile(directory, ".aks-node-controller-hotfix-stage-*", payload, mode) - if err != nil { - return stagedEntry{}, false, err - } - staged := stagedEntry{ - destination: destination, - stagedPath: stagedPath, - originalExist: true, - } - backupPath, err := writeTempFile( - directory, - ".aks-node-controller-hotfix-backup-*", - current, - originalMode, - ) - if err != nil { - _ = os.Remove(stagedPath) - return stagedEntry{}, false, fmt.Errorf("stage destination backup: %w", err) - } - staged.backupPath = backupPath - return staged, true, nil -} - -func writeTempFile(directory, pattern string, content []byte, mode os.FileMode) (string, error) { - temp, err := os.CreateTemp(directory, pattern) - if err != nil { - return "", fmt.Errorf("create temporary file: %w", err) - } - tempPath := temp.Name() - cleanup := func() { - _ = temp.Close() - _ = os.Remove(tempPath) - } - if _, err := temp.Write(content); err != nil { - cleanup() - return "", fmt.Errorf("write temporary file: %w", err) - } - if err := temp.Sync(); err != nil { - cleanup() - return "", fmt.Errorf("sync temporary file: %w", err) - } - if err := temp.Close(); err != nil { - cleanup() - return "", fmt.Errorf("close temporary file: %w", err) - } - if err := os.Chmod(tempPath, mode); err != nil { - cleanup() - return "", fmt.Errorf("chmod temporary file: %w", err) - } - return tempPath, nil -} - -func commitStaged(staged []*stagedEntry) error { - return commitStagedWithRename(staged, os.Rename) -} - -func commitStagedWithRename(staged []*stagedEntry, rename func(string, string) error) error { - committed := 0 - defer cleanupStaged(staged) - for index, entry := range staged { - if err := rename(entry.stagedPath, entry.destination); err != nil { - rollbackErr := rollbackStaged(staged[:committed], rename) - if rollbackErr != nil { - return fmt.Errorf( - "commit hotfix destination %s: %w; rollback failed: %w", - entry.destination, - err, - rollbackErr, - ) - } - return fmt.Errorf("commit hotfix destination %s: %w", entry.destination, err) - } - staged[index].stagedPath = "" - committed++ - } - return nil -} - -func rollbackStaged(committed []*stagedEntry, rename func(string, string) error) error { - var rollbackErrors []error - for index := len(committed) - 1; index >= 0; index-- { - entry := committed[index] - var err error - if entry.originalExist { - err = rename(entry.backupPath, entry.destination) - if err == nil { - entry.backupPath = "" - } - } else { - err = os.Remove(entry.destination) - } - if err != nil { - entry.preserveBackup = true - rollbackErrors = append( - rollbackErrors, - fmt.Errorf( - "restore %s from preserved backup %s: %w", - entry.destination, - entry.backupPath, - err, - ), - ) - } - } - return errors.Join(rollbackErrors...) -} - -func cleanupStaged(staged []*stagedEntry) { - for _, entry := range staged { - if entry.stagedPath != "" { - _ = os.Remove(entry.stagedPath) - } - if entry.backupPath != "" && !entry.preserveBackup { - _ = os.Remove(entry.backupPath) - } - } -} diff --git a/aks-node-controller/scripthotfix/applier_test.go b/aks-node-controller/scripthotfix/applier_test.go deleted file mode 100644 index dd3a2256847..00000000000 --- a/aks-node-controller/scripthotfix/applier_test.go +++ /dev/null @@ -1,301 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. - -package scripthotfix - -import ( - "bytes" - "compress/gzip" - "encoding/base64" - "errors" - "os" - "path/filepath" - "runtime" - "testing" - "testing/fstest" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "gopkg.in/yaml.v3" -) - -func TestClassifyPlatform(t *testing.T) { - tests := []struct { - name string - release string - expected Platform - }{ - {name: "Ubuntu", release: "ID=ubuntu\n", expected: PlatformUbuntu}, - {name: "Mariner", release: "ID=mariner\n", expected: PlatformMariner}, - {name: "Azure Linux", release: "ID=azurelinux\n", expected: PlatformMariner}, - { - name: "OS Guard variant wins over Azure Linux ID", - release: "ID=azurelinux\nVARIANT_ID=osguard\n", - expected: PlatformAzlOSGuard, - }, - { - name: "ACL variant wins over Azure Linux ID", - release: "ID=azurelinux\nVARIANT_ID=azurecontainerlinux\n", - expected: PlatformACL, - }, - {name: "ACL dedicated ID", release: "ID=azurecontainerlinux\n", expected: PlatformACL}, - {name: "Flatcar", release: "ID=flatcar\n", expected: PlatformFlatcar}, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - releasePath := filepath.Join(t.TempDir(), "os-release") - require.NoError(t, os.WriteFile(releasePath, []byte(test.release), 0o600)) - - actual, err := ClassifyPlatform(releasePath) - - require.NoError(t, err) - assert.Equal(t, test.expected, actual) - }) - } - - t.Run("unsupported ID fails explicitly", func(t *testing.T) { - releasePath := filepath.Join(t.TempDir(), "os-release") - require.NoError(t, os.WriteFile(releasePath, []byte("ID=other\n"), 0o600)) - - _, err := ClassifyPlatform(releasePath) - - require.ErrorContains(t, err, "unsupported OS ID") - }) -} - -func TestApplyEmbeddedInactivePayloadDoesNotReadOSRelease(t *testing.T) { - original := generatedFiles - generatedFiles = fstest.MapFS{ - "generated/active": &fstest.MapFile{Data: []byte("false\n")}, - } - t.Cleanup(func() { - generatedFiles = original - }) - - result, err := ApplyEmbedded(filepath.Join(t.TempDir(), "missing-os-release")) - - require.NoError(t, err) - assert.Equal(t, Result{}, result) -} - -func TestApplyFSUsesSelectedRenderedNodeCustomData(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("Windows rename cannot atomically replace an existing destination") - } - - directory := t.TempDir() - destination := filepath.Join(directory, "provision.sh") - require.NoError(t, os.WriteFile(destination, []byte("old"), 0o600)) - payload := []byte("#!/bin/sh\necho fixed\n") - files := renderedFS(t, PlatformUbuntu, []writeFile{{ - Path: destination, - Permissions: "0744", - Encoding: "base64", - Owner: "root", - Content: base64.StdEncoding.EncodeToString(payload), - }}) - - first, err := applyFS(files, PlatformUbuntu) - - require.NoError(t, err) - assert.Equal(t, Result{Applied: 1}, first) - actual, err := os.ReadFile(destination) - require.NoError(t, err) - assert.Equal(t, payload, actual) - info, err := os.Stat(destination) - require.NoError(t, err) - assert.Equal(t, os.FileMode(0o744), info.Mode().Perm()) - - second, err := applyFS(files, PlatformUbuntu) - - require.NoError(t, err) - assert.Equal(t, Result{Skipped: 1}, second) -} - -func TestApplyFSSkipsMissingDestination(t *testing.T) { - destination := filepath.Join(t.TempDir(), "missing.sh") - files := renderedFS(t, PlatformMariner, []writeFile{{ - Path: destination, - Permissions: "0744", - Owner: "root", - Content: "hotfix", - }}) - - result, err := applyFS(files, PlatformMariner) - - require.NoError(t, err) - assert.Equal(t, Result{Skipped: 1}, result) - _, statErr := os.Stat(destination) - assert.True(t, os.IsNotExist(statErr)) -} - -func TestRenderedNodeCustomDataValidation(t *testing.T) { - validDestination := filepath.Join(t.TempDir(), "provision.sh") - valid := writeFile{ - Path: validDestination, - Permissions: "0744", - Owner: "root", - Content: "hotfix", - } - - tests := []struct { - name string - files []writeFile - expectedErr string - }{ - { - name: "unsafe destination", - files: []writeFile{{ - Path: "../provision.sh", - Permissions: "0744", - Owner: "root", - Content: "hotfix", - }}, - expectedErr: "unsafe destination", - }, - { - name: "destination with embedded backslash", - files: []writeFile{{ - Path: `/tmp/provision\script.sh`, - Permissions: "0744", - Owner: "root", - Content: "hotfix", - }}, - expectedErr: "backslashes are not allowed", - }, - { - name: "invalid mode", - files: []writeFile{{ - Path: validDestination, - Permissions: "0999", - Owner: "root", - Content: "hotfix", - }}, - expectedErr: "invalid mode", - }, - { - name: "unsupported owner", - files: []writeFile{{ - Path: validDestination, - Permissions: "0744", - Owner: "nobody", - Content: "hotfix", - }}, - expectedErr: "unsupported owner", - }, - { - name: "duplicate destination", - files: []writeFile{valid, valid}, - expectedErr: "duplicate destination", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - _, err := loadAndValidate(renderedFS(t, PlatformUbuntu, test.files), PlatformUbuntu) - require.ErrorContains(t, err, test.expectedErr) - }) - } - - t.Run("unknown YAML field", func(t *testing.T) { - files := fstest.MapFS{ - renderedPath(PlatformUbuntu): &fstest.MapFile{ - Data: []byte("write_files: []\nunknown: true\n"), - }, - } - _, err := loadAndValidate(files, PlatformUbuntu) - require.ErrorContains(t, err, "field unknown not found") - }) -} - -func TestDecodeContentGzip(t *testing.T) { - var compressed bytes.Buffer - writer := gzip.NewWriter(&compressed) - _, err := writer.Write([]byte("rendered payload")) - require.NoError(t, err) - require.NoError(t, writer.Close()) - - decoded, err := decodeContent(writeFile{ - Encoding: "gzip", - Content: compressed.String(), - }) - - require.NoError(t, err) - assert.Equal(t, []byte("rendered payload"), decoded) -} - -func TestApplyFSDoesNotCommitWhenLaterEntryCannotBeStaged(t *testing.T) { - directory := t.TempDir() - firstDestination := filepath.Join(directory, "first.sh") - require.NoError(t, os.WriteFile(firstDestination, []byte("original"), 0o700)) - files := renderedFS(t, PlatformUbuntu, []writeFile{ - { - Path: firstDestination, - Permissions: "0744", - Owner: "root", - Content: "first hotfix", - }, - { - Path: directory, - Permissions: "0744", - Owner: "root", - Content: "second hotfix", - }, - }) - - _, err := applyFS(files, PlatformUbuntu) - - require.ErrorContains(t, err, "read destination") - actual, readErr := os.ReadFile(firstDestination) - require.NoError(t, readErr) - assert.Equal(t, []byte("original"), actual) -} - -func TestCommitStagedRollsBackEarlierReplacement(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("Windows rename cannot atomically replace an existing destination") - } - directory := t.TempDir() - firstDestination := filepath.Join(directory, "first.sh") - secondDestination := filepath.Join(directory, "second.sh") - require.NoError(t, os.WriteFile(firstDestination, []byte("first original"), 0o700)) - require.NoError(t, os.WriteFile(secondDestination, []byte("second original"), 0o711)) - first, changed, err := stageEntry(firstDestination, []byte("first hotfix"), 0o744) - require.NoError(t, err) - require.True(t, changed) - second, changed, err := stageEntry(secondDestination, []byte("second hotfix"), 0o755) - require.NoError(t, err) - require.True(t, changed) - - err = commitStagedWithRename( - []*stagedEntry{&first, &second}, - func(source string, destination string) error { - if source == second.stagedPath { - return errors.New("injected rename failure") - } - return os.Rename(source, destination) - }, - ) - - require.ErrorContains(t, err, "injected rename failure") - firstActual, readErr := os.ReadFile(firstDestination) - require.NoError(t, readErr) - assert.Equal(t, []byte("first original"), firstActual) - secondActual, readErr := os.ReadFile(secondDestination) - require.NoError(t, readErr) - assert.Equal(t, []byte("second original"), secondActual) -} - -func renderedFS(t *testing.T, platform Platform, files []writeFile) fstest.MapFS { - t.Helper() - data, err := yaml.Marshal(nodeCustomData{WriteFiles: files}) - require.NoError(t, err) - return fstest.MapFS{ - renderedPath(platform): &fstest.MapFile{Data: data}, - } -} - -func renderedPath(platform Platform) string { - return "generated/rendered_nodecustomdata_" + string(platform) + ".yml" -} diff --git a/hotfix/hotfix_generate.py b/hotfix/hotfix_generate.py index 2b673e648d1..e59056e97b2 100644 --- a/hotfix/hotfix_generate.py +++ b/hotfix/hotfix_generate.py @@ -65,42 +65,13 @@ # CSE main / start "cse_main.sh": "provisionScript", "cse_start.sh": "provisionStartScript", - # Python scripts - "cse_redact_cloud_config.py": "provisionRedactCloudConfig", - "cse_send_logs.py": "provisionSendLogs", - # Other scripts - "reconcile-private-hosts.sh": "reconcilePrivateHostsScript", - "bind-mount.sh": "bindMountScript", - "mig-partition.sh": "migPartitionScript", - "enable-dhcpv6.sh": "dhcpv6ConfigurationScript", - "ensure_imds_restriction.sh": "ensureIMDSRestrictionScript", - "ensure-no-dup.sh": "ensureNoDupEbtablesScript", - "cloud-init-status-check.sh": "cloudInitStatusCheckScript", - "measure-tls-bootstrapping-latency.sh": "measureTLSBootstrappingLatencyScript", - "validate-kubelet-credentials.sh": "validateKubeletCredentialsScript", - "setup-custom-search-domains.sh": "customSearchDomainsScript", + # Other scripts present in traditional nodecustomdata "configure-azure-network.sh": "configureAzureNetworkScript", - "init-aks-custom-cloud.sh": "initAKSCustomCloud", "init-aks-cloud.sh": "initAKSCloud", - # Distro-specific scripts - "ubuntu/ubuntu-snapshot-update.sh": "snapshotUpdateScript", - "mariner/mariner-package-update.sh": "packageUpdateScriptMariner", - # Systemd services + # Systemd files present in traditional nodecustomdata "kubelet.service": "kubeletSystemdService", "reconcile-private-hosts.service": "reconcilePrivateHostsService", - "bind-mount.service": "bindMountSystemdService", - "dhcpv6.service": "dhcpv6SystemdService", - "mig-partition.service": "migPartitionSystemdService", - "secure-tls-bootstrap.service": "secureTLSBootstrapService", - "ensure-no-dup.service": "ensureNoDupEbtablesService", - "measure-tls-bootstrapping-latency.service": "measureTLSBootstrappingLatencyService", - "ubuntu/snapshot-update.service": "snapshotUpdateService", - "ubuntu/snapshot-update.timer": "snapshotUpdateTimer", - "mariner/package-update.service": "packageUpdateServiceMariner", - "mariner/package-update.timer": "packageUpdateTimerMariner", "99-azure-network.rules": "azureNetworkUdevRule", - # Component manifest - "manifest.json": "componentManifestFile", } # Distro-variant variable keys that share a single conditional write_files block. @@ -195,28 +166,19 @@ def path_changed(base_ref, *paths): def write_hotfix_file(version): - """Write the resolved ANC version to TARGET_FILE when active. - - When no hotfix applies, remove TARGET_FILE if present. An empty JSON object is - still embedded as a real scriptless customData file, which changes payload - shape even though there is no hotfix for the wrapper to consume. - """ - payload = {} - if version: - payload["version"] = version - - if payload: - with open(TARGET_FILE, "w") as f: - json.dump(payload, f, indent=4) - f.write("\n") - print(f"Wrote {payload} to {TARGET_FILE}", file=sys.stderr) + """Write a new ANC version while retaining any active inherited pointer.""" + if not version: + print( + f"No new ANC hotfix version; preserving {TARGET_FILE} if present", + file=sys.stderr, + ) return - try: - os.remove(TARGET_FILE) - print(f"No active hotfix; removed {TARGET_FILE}", file=sys.stderr) - except FileNotFoundError: - print(f"No active hotfix; {TARGET_FILE} already absent", file=sys.stderr) + payload = {"version": version} + with open(TARGET_FILE, "w") as f: + json.dump(payload, f, indent=4) + f.write("\n") + print(f"Wrote {payload} to {TARGET_FILE}", file=sys.stderr) def detect_changed_varkeys(base_ref, available_varkeys=None): @@ -248,6 +210,11 @@ def detect_changed_varkeys(base_ref, available_varkeys=None): f"changed hotfix source {local_path} does not exist at {source_path}" ) varkey = SOURCE_TO_VARKEY[local_path] + if available_varkeys is not None and varkey not in available_varkeys: + raise GenerationError( + f"changed hotfix source {local_path} maps to {varkey}, " + "which has no traditional nodecustomdata write_files entry" + ) matched_varkeys.add(varkey) if varkey in VARKEY_TO_BLOCK_GROUP: matched_block_groups.add(VARKEY_TO_BLOCK_GROUP[varkey]) @@ -398,6 +365,13 @@ def build_hotfix_template(target_varkeys, traditional_lines): def write_rendered_payload(target_varkeys, traditional_lines): """Render platform-specific YAML through AgentBaker's production template path.""" + if not target_varkeys: + print( + f"No new script hotfixes; preserving {GENERATED_DIR}", + file=sys.stderr, + ) + return + hotfix_template = build_hotfix_template(target_varkeys, traditional_lines) shutil.rmtree(GENERATED_DIR, ignore_errors=True) os.makedirs(GENERATED_DIR, exist_ok=True) @@ -423,8 +397,9 @@ def write_rendered_payload(target_varkeys, traditional_lines): os.remove(template_path) except FileNotFoundError: pass + with open(os.path.join(GENERATED_DIR, "active"), "w", newline="\n") as active_file: - active_file.write("true\n" if target_varkeys else "false\n") + active_file.write("true\n") print( f"Rendered {len(target_varkeys)} hotfix variable keys into {GENERATED_DIR}", file=sys.stderr, diff --git a/hotfix/hotfix_generate_test.py b/hotfix/hotfix_generate_test.py index 0ee23e37cd9..591d2c245a8 100644 --- a/hotfix/hotfix_generate_test.py +++ b/hotfix/hotfix_generate_test.py @@ -199,7 +199,38 @@ def render(command, check): self.assertFalse((generated / ".nodecustomdata-hotfix.template").exists()) self.assertEqual("true\n", (generated / "active").read_text()) - def test_write_hotfix_file_contains_only_anc_version(self): + def test_write_rendered_payload_preserves_previous_hotfix_when_unchanged(self): + with tempfile.TemporaryDirectory() as temp_dir: + generated = Path(temp_dir) / "generated" + generated.mkdir() + (generated / "active").write_text("true\n") + platforms = ("ubuntu", "mariner", "acl", "azlosguard", "flatcar") + for platform in platforms: + (generated / f"rendered_nodecustomdata_{platform}.yml").write_text( + f"write_files:\n- path: /{platform}-existing\n" + ) + + with mock.patch.object( + hotfix_generate, "GENERATED_DIR", str(generated) + ), mock.patch.object( + hotfix_generate.subprocess, "run" + ) as run: + hotfix_generate.write_rendered_payload( + set(), + TRADITIONAL_TEMPLATE.splitlines(keepends=True), + ) + + run.assert_not_called() + self.assertEqual("true\n", (generated / "active").read_text()) + for platform in platforms: + self.assertIn( + f"/{platform}-existing", + ( + generated / f"rendered_nodecustomdata_{platform}.yml" + ).read_text(), + ) + + def test_write_hotfix_file_contains_only_anc_version_and_preserves_it(self): with tempfile.TemporaryDirectory() as temp_dir: target = Path(temp_dir) / "hotfix.json" with mock.patch.object( @@ -210,6 +241,18 @@ def test_write_hotfix_file_contains_only_anc_version(self): {"version": "202608.14.1"}, json.loads(target.read_text()), ) + hotfix_generate.write_hotfix_file("") + self.assertEqual( + {"version": "202608.14.1"}, + json.loads(target.read_text()), + ) + + def test_write_hotfix_file_without_version_keeps_missing_target_absent(self): + with tempfile.TemporaryDirectory() as temp_dir: + target = Path(temp_dir) / "hotfix.json" + with mock.patch.object( + hotfix_generate, "TARGET_FILE", str(target) + ): hotfix_generate.write_hotfix_file("") self.assertFalse(target.exists()) @@ -233,6 +276,35 @@ def test_unmapped_hotfixable_script_fails(self): ): hotfix_generate.detect_changed_varkeys("base") + def test_mapped_source_without_renderable_entry_fails(self): + with tempfile.TemporaryDirectory() as temp_dir: + changed = Path(temp_dir) / "mapped.sh" + changed.write_text("#!/bin/sh") + result = subprocess.CompletedProcess( + args=[], + returncode=0, + stdout=f"{changed}\n", + ) + with mock.patch.object( + hotfix_generate, "ARTIFACTS_DIR", temp_dir + ), mock.patch.object( + hotfix_generate, + "SOURCE_TO_VARKEY", + {"mapped.sh": "missingVariable"}, + ), mock.patch.object( + hotfix_generate.subprocess, + "run", + return_value=result, + ): + with self.assertRaisesRegex( + hotfix_generate.GenerationError, + "has no traditional nodecustomdata write_files entry", + ): + hotfix_generate.detect_changed_varkeys( + "base", + available_varkeys={"otherVariable"}, + ) + if __name__ == "__main__": unittest.main() From f71bad0fbfc79a8f575c6246b92c6fc4436c47eb Mon Sep 17 00:00:00 2001 From: Devin Wong Date: Thu, 3 Sep 2026 13:22:11 -0700 Subject: [PATCH 03/26] refactor: clarify embedded hotfix activation Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- aks-node-controller/app.go | 2 +- aks-node-controller/embeddednodecustomdata.go | 2 +- aks-node-controller/embeddednodecustomdata_test.go | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/aks-node-controller/app.go b/aks-node-controller/app.go index d92dc845f27..24f9feba18e 100644 --- a/aks-node-controller/app.go +++ b/aks-node-controller/app.go @@ -689,7 +689,7 @@ func (a *App) Provision(ctx context.Context, flags ProvisionFlags) (*ProvisionRe func (a *App) applyEmbeddedHotfixPayload() { applyEmbeddedHotfix := a.applyEmbeddedHotfix if applyEmbeddedHotfix == nil { - applyEmbeddedHotfix = applyEmbeddedNodeCustomData + applyEmbeddedHotfix = applyEmbeddedNodeCustomDataIfActive } result, err := applyEmbeddedHotfix(a.osReleasePath) if err != nil { diff --git a/aks-node-controller/embeddednodecustomdata.go b/aks-node-controller/embeddednodecustomdata.go index fe2aa7479a3..0044edefe7f 100644 --- a/aks-node-controller/embeddednodecustomdata.go +++ b/aks-node-controller/embeddednodecustomdata.go @@ -26,7 +26,7 @@ var embeddedGeneratedNodeCustomData embed.FS var generatedNodeCustomData fs.FS = embeddedGeneratedNodeCustomData -func applyEmbeddedNodeCustomData(osReleasePath string) (nodeCustomDataApplyResult, error) { +func applyEmbeddedNodeCustomDataIfActive(osReleasePath string) (nodeCustomDataApplyResult, error) { active, err := fs.ReadFile(generatedNodeCustomData, "scripthotfix/generated/active") if err != nil { return nodeCustomDataApplyResult{}, fmt.Errorf("read embedded hotfix state: %w", err) diff --git a/aks-node-controller/embeddednodecustomdata_test.go b/aks-node-controller/embeddednodecustomdata_test.go index c5263983092..e50392d46e5 100644 --- a/aks-node-controller/embeddednodecustomdata_test.go +++ b/aks-node-controller/embeddednodecustomdata_test.go @@ -61,7 +61,7 @@ func TestClassifyNodeCustomDataPlatform(t *testing.T) { }) } -func TestApplyEmbeddedNodeCustomDataInactiveDoesNotReadOSRelease(t *testing.T) { +func TestApplyEmbeddedNodeCustomDataIfActiveSkipsInactivePayload(t *testing.T) { original := generatedNodeCustomData generatedNodeCustomData = fstest.MapFS{ "scripthotfix/generated/active": &fstest.MapFile{Data: []byte("false\n")}, @@ -70,13 +70,13 @@ func TestApplyEmbeddedNodeCustomDataInactiveDoesNotReadOSRelease(t *testing.T) { generatedNodeCustomData = original }) - result, err := applyEmbeddedNodeCustomData(filepath.Join(t.TempDir(), "missing-os-release")) + result, err := applyEmbeddedNodeCustomDataIfActive(filepath.Join(t.TempDir(), "missing-os-release")) require.NoError(t, err) assert.Equal(t, nodeCustomDataApplyResult{}, result) } -func TestApplyEmbeddedNodeCustomDataSelectsPlatformPayload(t *testing.T) { +func TestApplyEmbeddedNodeCustomDataIfActiveSelectsPlatformPayload(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("Windows rename cannot atomically replace an existing destination") } @@ -103,7 +103,7 @@ func TestApplyEmbeddedNodeCustomDataSelectsPlatformPayload(t *testing.T) { generatedNodeCustomData = original }) - result, err := applyEmbeddedNodeCustomData(releasePath) + result, err := applyEmbeddedNodeCustomDataIfActive(releasePath) require.NoError(t, err) assert.Equal(t, nodeCustomDataApplyResult{Applied: 1}, result) From b9a811a8aa57ac756d5541134a02ea8d2fe299b2 Mon Sep 17 00:00:00 2001 From: Devin Wong Date: Thu, 3 Sep 2026 15:09:47 -0700 Subject: [PATCH 04/26] fix: resolve golangci-lint failures in ANC hotfix embedding - errorlint: use errors.Is(err, io.EOF) for wrapped-error safety - govet shadow: rename shadowed err to mkErr in stageNodeCustomDataEntry - gochecknoglobals: annotate generatedNodeCustomData test-injection global - gochecknoglobals/gosec: scope platforms slice to main() and tighten rendered-file WriteFile perms to 0o600 - rename applyEmbeddedHotfixPayload to applyEmbeddedHotfixIfNeeded and document newRenderConfig placeholder fields per review feedback Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- aks-node-controller/app.go | 4 +-- aks-node-controller/embeddednodecustomdata.go | 1 + aks-node-controller/nodecustomdata.go | 6 ++--- hotfix/render-nodecustomdata/main.go | 25 ++++++++++++------- 4 files changed, 22 insertions(+), 14 deletions(-) diff --git a/aks-node-controller/app.go b/aks-node-controller/app.go index 24f9feba18e..b658e730650 100644 --- a/aks-node-controller/app.go +++ b/aks-node-controller/app.go @@ -686,7 +686,7 @@ func (a *App) Provision(ctx context.Context, flags ProvisionFlags) (*ProvisionRe return provisionResult, err } -func (a *App) applyEmbeddedHotfixPayload() { +func (a *App) applyEmbeddedHotfixIfNeeded() { applyEmbeddedHotfix := a.applyEmbeddedHotfix if applyEmbeddedHotfix == nil { applyEmbeddedHotfix = applyEmbeddedNodeCustomDataIfActive @@ -729,7 +729,7 @@ func (a *App) runProvision(ctx context.Context, flags ProvisionFlags, dryRun boo if dryRun { a.cmdRun = cmdRunnerDryRun } else { - a.applyEmbeddedHotfixPayload() + a.applyEmbeddedHotfixIfNeeded() } return a.Provision(ctx, flags) } diff --git a/aks-node-controller/embeddednodecustomdata.go b/aks-node-controller/embeddednodecustomdata.go index 0044edefe7f..c26b8d8134a 100644 --- a/aks-node-controller/embeddednodecustomdata.go +++ b/aks-node-controller/embeddednodecustomdata.go @@ -24,6 +24,7 @@ const ( //go:embed scripthotfix/generated var embeddedGeneratedNodeCustomData embed.FS +//nolint:gochecknoglobals // indirection point so tests can inject an alternate filesystem var generatedNodeCustomData fs.FS = embeddedGeneratedNodeCustomData func applyEmbeddedNodeCustomDataIfActive(osReleasePath string) (nodeCustomDataApplyResult, error) { diff --git a/aks-node-controller/nodecustomdata.go b/aks-node-controller/nodecustomdata.go index de4db972e17..5820602bbd2 100644 --- a/aks-node-controller/nodecustomdata.go +++ b/aks-node-controller/nodecustomdata.go @@ -125,7 +125,7 @@ func parseNodeCustomData(data []byte, options nodeCustomDataApplyOptions) (nodeC return nodeCustomData{}, fmt.Errorf("decode nodecustomdata %s: %w", options.source, err) } var trailing any - if err := decoder.Decode(&trailing); err != io.EOF { + if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { return nodeCustomData{}, fmt.Errorf("nodecustomdata %s has trailing content", options.source) } return customData, nil @@ -257,8 +257,8 @@ func stageNodeCustomDataEntry(entry nodeCustomDataEntry, replaceOnly bool) (stag if !info.IsDir() { return stagedNodeCustomDataEntry{}, false, fmt.Errorf("destination parent %s is not a directory", directory) } - } else if err := os.MkdirAll(directory, 0o755); err != nil { - return stagedNodeCustomDataEntry{}, false, fmt.Errorf("create parent directory: %w", err) + } else if mkErr := os.MkdirAll(directory, 0o755); mkErr != nil { + return stagedNodeCustomDataEntry{}, false, fmt.Errorf("create parent directory: %w", mkErr) } stagedPath, err := writeNodeCustomDataTempFile(directory, ".aks-node-controller-nodecustomdata-stage-*", entry.content, entry.mode) diff --git a/hotfix/render-nodecustomdata/main.go b/hotfix/render-nodecustomdata/main.go index f9232ea0a58..c2ea11c11d5 100644 --- a/hotfix/render-nodecustomdata/main.go +++ b/hotfix/render-nodecustomdata/main.go @@ -18,15 +18,15 @@ type platform struct { distro datamodel.Distro } -var platforms = []platform{ - {name: "ubuntu", distro: datamodel.AKSUbuntuContainerd2204Gen2}, - {name: "mariner", distro: datamodel.AKSAzureLinuxV3Gen2}, - {name: "acl", distro: datamodel.AKSACLGen2TL}, - {name: "azlosguard", distro: datamodel.AKSAzureLinuxV3OSGuardGen2FIPSTL}, - {name: "flatcar", distro: datamodel.AKSFlatcarGen2}, -} - func main() { + platforms := []platform{ + {name: "ubuntu", distro: datamodel.AKSUbuntuContainerd2204Gen2}, + {name: "mariner", distro: datamodel.AKSAzureLinuxV3Gen2}, + {name: "acl", distro: datamodel.AKSACLGen2TL}, + {name: "azlosguard", distro: datamodel.AKSAzureLinuxV3OSGuardGen2FIPSTL}, + {name: "flatcar", distro: datamodel.AKSFlatcarGen2}, + } + templatePath := flag.String("template", "", "path to the hotfix nodecustomdata template") outputDir := flag.String("output-dir", "", "directory for rendered nodecustomdata files") flag.Parse() @@ -56,12 +56,19 @@ func main() { *outputDir, "rendered_nodecustomdata_"+target.name+".yml", ) - if err := os.WriteFile(outputPath, []byte(rendered), 0o644); err != nil { + if err := os.WriteFile(outputPath, []byte(rendered), 0o600); err != nil { fatalf("write %s nodecustomdata: %v", target.name, err) } } } +// newRenderConfig builds the minimal NodeBootstrappingConfiguration required to +// run agent.RenderLinuxNodeCustomDataTemplate. Only the Distro selector affects +// the hotfix write_files blocks we render; the remaining fields (orchestrator +// version, location, FQDN, KubernetesConfig, etc.) are placeholders whose sole +// purpose is to satisfy the renderer's unconditional dereferences of +// OrchestratorProfile/KubernetesConfig so rendering doesn't nil-panic. Their +// values are not meaningful and are not consumed by the embedded output. func newRenderConfig(distro datamodel.Distro) *datamodel.NodeBootstrappingConfiguration { profile := &datamodel.AgentPoolProfile{ Name: "hotfix-render", From 729c5905d90458ac01e3a3e2a8a9118b72e81b83 Mon Sep 17 00:00:00 2001 From: Devin Wong Date: Thu, 3 Sep 2026 15:36:21 -0700 Subject: [PATCH 05/26] fix(hotfix): diff scripts against frozen VHD baseline for cumulative payloads detect_changed_varkeys previously diffed against the moving base branch tip, so a later hotfix only re-rendered its own script and silently dropped an earlier hotfix's rendered block (non-cumulative regression). Diff against the immutable VHD baseline tag derived from linux_sig_version.json instead, so every generated payload re-renders all scripts changed since the VHD. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- hotfix/hotfix_generate.py | 74 ++++++++++++++++++++++++++++------ hotfix/hotfix_generate_test.py | 62 ++++++++++++++++++++++++++-- 2 files changed, 121 insertions(+), 15 deletions(-) diff --git a/hotfix/hotfix_generate.py b/hotfix/hotfix_generate.py index e59056e97b2..e95fcacc896 100644 --- a/hotfix/hotfix_generate.py +++ b/hotfix/hotfix_generate.py @@ -8,17 +8,23 @@ testdata files vs the base branch, bumps the patch of the current pkg/agent/datamodel/linux_sig_version.json version and uses it as `version`. -2. Detects which CSE provisioning scripts changed vs the base branch, selects their - write_files entries from parts/linux/cloud-init/nodecustomdata.yml, and renders - self-contained ANC payloads for each Linux platform with AgentBaker's canonical - Go-template renderer. +2. Detects which CSE provisioning scripts differ from the immutable VHD baseline + (the release tag the VHD was built from, derived from linux_sig_version.json), + selects their write_files entries from parts/linux/cloud-init/nodecustomdata.yml, + and renders self-contained ANC payloads for each Linux platform with AgentBaker's + canonical Go-template renderer. Diffing against the frozen baseline (rather than + the moving base branch) keeps every generated payload cumulative: a later hotfix + re-renders all scripts changed since the VHD, so it never silently drops an + earlier hotfix's script. 3. Writes the resolved ANC `version` to parts/linux/cloud-init/artifacts/aks-node-controller-hotfix.json when active. Usage: python3 hotfix/hotfix_generate.py - base_ref: git ref to diff against for changed-script/changed-code detection - (e.g., origin/official/v20260219) + base_ref: git ref for the PR base branch, used only to detect ANC Go-module + changes for the version bump (e.g., origin/official/v20260219). The + changed-script detection instead diffs against the VHD baseline tag + derived from linux_sig_version.json. This script is called by the hotfix-generate GH Action. """ @@ -155,6 +161,43 @@ def bump_version(base_version): patch += 1 +def baseline_tag(base_version): + """Return the immutable AgentBaker tag for the VHD baseline scripts. + + base_version is 'YYYYMM.DD.PATCH' (from linux_sig_version.json, frozen on an + official/* branch once cut). The matching tag is 'v0.YYYYMMDD.PATCH', which is + the commit the VHD was built from, so parts/linux/cloud-init/artifacts/ at that + tag holds exactly the scripts baked into the VHD. + """ + match = re.match(r'^(\d{6})\.(\d{2})\.(\d+)$', base_version) + if not match: + raise GenerationError(f"invalid baseline version '{base_version}'") + yyyymm, dd, patch = match.group(1), match.group(2), match.group(3) + return f"v0.{yyyymm}{dd}.{patch}" + + +def resolve_baseline_ref(base_version): + """Resolve the VHD baseline git ref, fetching the tag if needed. + + Script hotfixes are cumulative, so changed-script detection diffs against the + frozen VHD baseline tag rather than the moving base branch. Raise if the tag + cannot be resolved, since diffing against a missing ref would silently produce + a non-cumulative (or empty) payload. + """ + tag = baseline_tag(base_version) + # Best-effort fetch of just this tag in case the checkout did not include it. + subprocess.run( + ["git", "fetch", "--quiet", "--no-tags", "origin", "tag", tag], + capture_output=True, + ) + if not tag_exists(tag): + raise GenerationError( + f"baseline tag {tag} (derived from {LINUX_SIG_VERSION_FILE}) is not " + "available; cannot compute the cumulative script hotfix set" + ) + return tag + + def path_changed(base_ref, *paths): """Return True if any selected path differs from the working tree and base_ref.""" result = subprocess.run(["git", "diff", "--quiet", base_ref, "--", *paths]) @@ -408,16 +451,25 @@ def write_rendered_payload(target_varkeys, traditional_lines): def main(): parser = argparse.ArgumentParser(description="Generate ANC hotfix assets") - parser.add_argument("base_ref", help="git ref to diff against") + parser.add_argument( + "base_ref", + help="git ref for the PR base branch, used to detect ANC module changes", + ) args = parser.parse_args() base_ref = args.base_ref - # Best-effort: make sure locally-known tags are up to date before checking for - # collisions. Ignore failures (e.g. no network) and fall back to local tags. + # Best-effort: make sure locally-known tags are up to date before resolving the + # baseline and checking for version collisions. Ignore failures (e.g. no + # network) and fall back to local tags. subprocess.run(["git", "fetch", "--tags"], capture_output=True) + base_version = read_base_version() + try: validate_source_mappings() + # Diff changed scripts against the frozen VHD baseline (not the moving base + # branch) so the rendered payload stays cumulative across hotfixes. + baseline_ref = resolve_baseline_ref(base_version) with open(TEMPLATE, "r") as template_file: template_lines = template_file.readlines() _, else_line, end_line = find_block_boundaries(template_lines) @@ -430,7 +482,7 @@ def main(): for varkeys, _ in parse_write_files_blocks(traditional_lines): available_varkeys.update(varkeys) changed_varkeys = detect_changed_varkeys( - base_ref, + baseline_ref, available_varkeys=available_varkeys, ) write_rendered_payload(changed_varkeys, traditional_lines) @@ -438,8 +490,6 @@ def main(): print(f"ERROR: {err}", file=sys.stderr) sys.exit(1) - base_version = read_base_version() - version = "" if path_changed( base_ref, diff --git a/hotfix/hotfix_generate_test.py b/hotfix/hotfix_generate_test.py index 591d2c245a8..8a34ee0f002 100644 --- a/hotfix/hotfix_generate_test.py +++ b/hotfix/hotfix_generate_test.py @@ -124,7 +124,7 @@ def test_detect_changed_varkeys_expands_distro_group(self): result = subprocess.CompletedProcess( args=[], returncode=0, - stdout=f"{changed}\n", + stdout=f"{temp_dir}/ubuntu/cse_helpers_ubuntu.sh\n", ) available = { "provisionSourceUbuntu", @@ -263,7 +263,7 @@ def test_unmapped_hotfixable_script_fails(self): result = subprocess.CompletedProcess( args=[], returncode=0, - stdout=f"{changed}\n", + stdout=f"{temp_dir}/unmapped.sh\n", ) with mock.patch.object( hotfix_generate, "ARTIFACTS_DIR", temp_dir @@ -283,7 +283,7 @@ def test_mapped_source_without_renderable_entry_fails(self): result = subprocess.CompletedProcess( args=[], returncode=0, - stdout=f"{changed}\n", + stdout=f"{temp_dir}/mapped.sh\n", ) with mock.patch.object( hotfix_generate, "ARTIFACTS_DIR", temp_dir @@ -306,5 +306,61 @@ def test_mapped_source_without_renderable_entry_fails(self): ) + def test_baseline_tag_derived_from_version(self): + self.assertEqual( + "v0.20260826.0", hotfix_generate.baseline_tag("202608.26.0") + ) + self.assertEqual( + "v0.20260702.3", hotfix_generate.baseline_tag("202607.02.3") + ) + + def test_baseline_tag_rejects_malformed_version(self): + with self.assertRaises(hotfix_generate.GenerationError): + hotfix_generate.baseline_tag("2026.7.1") + + def test_resolve_baseline_ref_requires_existing_tag(self): + with mock.patch.object( + hotfix_generate.subprocess, "run" + ), mock.patch.object(hotfix_generate, "tag_exists", return_value=False): + with self.assertRaises(hotfix_generate.GenerationError): + hotfix_generate.resolve_baseline_ref("202608.26.0") + + def test_resolve_baseline_ref_returns_tag_when_present(self): + with mock.patch.object( + hotfix_generate.subprocess, "run" + ), mock.patch.object(hotfix_generate, "tag_exists", return_value=True): + self.assertEqual( + "v0.20260826.0", + hotfix_generate.resolve_baseline_ref("202608.26.0"), + ) + + def test_detect_changed_varkeys_accumulates_all_scripts_since_baseline(self): + # A later hotfix must re-select every script that differs from the VHD + # baseline, not just the newest one, so the payload stays cumulative. + with tempfile.TemporaryDirectory() as temp_dir: + artifacts = Path(temp_dir) + for source in ("cse_config.sh", "cse_main.sh"): + (artifacts / source).write_text("hotfix") + result = subprocess.CompletedProcess( + args=[], + returncode=0, + stdout=( + f"{temp_dir}/cse_config.sh\n" + f"{temp_dir}/cse_main.sh\n" + ), + ) + available = {"provisionConfigs", "provisionScript"} + with mock.patch.object( + hotfix_generate, "ARTIFACTS_DIR", temp_dir + ), mock.patch.object( + hotfix_generate.subprocess, "run", return_value=result + ): + selected = hotfix_generate.detect_changed_varkeys( + "v0.20260826.0", + available_varkeys=available, + ) + self.assertEqual(available, selected) + + if __name__ == "__main__": unittest.main() From 4f9311ea49df2cc6524e374de847bea7b9e405ad Mon Sep 17 00:00:00 2001 From: Devin Wong Date: Thu, 3 Sep 2026 15:49:15 -0700 Subject: [PATCH 06/26] fix(anc): chmod embedded script temp file before fsync writeNodeCustomDataTempFile synced while the temp file still had CreateTemp's 0600 mode, then chmod'd after close with no re-sync. A crash after the later rename could durably leave the provisioning script non-executable. Chmod before Sync so the final mode is captured by the fsync. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- aks-node-controller/nodecustomdata.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/aks-node-controller/nodecustomdata.go b/aks-node-controller/nodecustomdata.go index 5820602bbd2..bc72ba5926e 100644 --- a/aks-node-controller/nodecustomdata.go +++ b/aks-node-controller/nodecustomdata.go @@ -302,6 +302,13 @@ func writeNodeCustomDataTempFile(directory, pattern string, content []byte, mode cleanup() return "", fmt.Errorf("write temporary file: %w", err) } + // Chmod before Sync so the final mode is included in the fsync; otherwise a + // crash after the later rename could durably leave the script with + // CreateTemp's 0600 (non-executable) mode. + if err := temp.Chmod(mode); err != nil { + cleanup() + return "", fmt.Errorf("chmod temporary file: %w", err) + } if err := temp.Sync(); err != nil { cleanup() return "", fmt.Errorf("sync temporary file: %w", err) @@ -310,10 +317,6 @@ func writeNodeCustomDataTempFile(directory, pattern string, content []byte, mode cleanup() return "", fmt.Errorf("close temporary file: %w", err) } - if err := os.Chmod(tempPath, mode); err != nil { - cleanup() - return "", fmt.Errorf("chmod temporary file: %w", err) - } return tempPath, nil } From 74868ce58e68e27c4dd1e5bf92e6826ad70043db Mon Sep 17 00:00:00 2001 From: Devin Wong Date: Thu, 3 Sep 2026 15:52:48 -0700 Subject: [PATCH 07/26] chore(anc): clearer detectPackageManager error for image-based ACL/Flatcar ACL and Flatcar have no apt/dnf/tdnf, so PMC package-based ANC self-update is intentionally unsupported there; they run VHD-baked ANC only. Give those IDs an explicit, self-documenting error instead of the generic 'unsupported OS'. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- aks-node-controller/hotfix.go | 5 +++++ aks-node-controller/hotfix_test.go | 12 ++++++++++++ 2 files changed, 17 insertions(+) diff --git a/aks-node-controller/hotfix.go b/aks-node-controller/hotfix.go index def7761611d..3fae8ed4ed0 100644 --- a/aks-node-controller/hotfix.go +++ b/aks-node-controller/hotfix.go @@ -273,6 +273,11 @@ func (a *App) detectPackageManager() (packageManager, error) { return pkgMgrApt, nil case "azurelinux", "mariner": return preferredRpmManager(), nil + case "azurecontainerlinux", "flatcar": + // ACL and Flatcar are image-based/immutable distros with no apt/dnf/tdnf. + // ANC self-update via a PMC package is intentionally unsupported there; + // they only ever run the ANC binary baked into the VHD. + return "", fmt.Errorf("PMC package-based ANC self-update is not supported on image-based OS %q", info.ID) default: return "", fmt.Errorf("unsupported OS: %s", info.ID) } diff --git a/aks-node-controller/hotfix_test.go b/aks-node-controller/hotfix_test.go index a9b283d7070..c8278a522e4 100644 --- a/aks-node-controller/hotfix_test.go +++ b/aks-node-controller/hotfix_test.go @@ -175,6 +175,18 @@ func TestDetectPackageManager(t *testing.T) { assert.Contains(t, err.Error(), "unsupported OS") }) + t.Run("image-based OS reports self-update unsupported", func(t *testing.T) { + for _, id := range []string{"azurecontainerlinux", "flatcar"} { + path := filepath.Join(t.TempDir(), "os-release") + require.NoError(t, os.WriteFile(path, []byte("ID="+id+"\n"), 0644)) + a := &App{osReleasePath: path} + _, err := a.detectPackageManager() + require.Error(t, err) + assert.Contains(t, err.Error(), "not supported on image-based OS") + assert.Contains(t, err.Error(), id) + } + }) + t.Run("missing ID line errors", func(t *testing.T) { path := filepath.Join(t.TempDir(), "os-release") require.NoError(t, os.WriteFile(path, []byte("VERSION_ID=1\n"), 0644)) From 06cdff1c335440ac651af52bc763d6ae6484b0ce Mon Sep 17 00:00:00 2001 From: Devin Wong Date: Fri, 4 Sep 2026 09:35:49 -0700 Subject: [PATCH 08/26] fix(anc): extract os-release ID constants to satisfy goconst Adding azurecontainerlinux/flatcar to detectPackageManager created a 3rd literal occurrence, tripping the goconst linter. Extract osReleaseIDAzureContainerLinux and osReleaseIDFlatcar constants and use them in both the classifier and the package-manager detector. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- aks-node-controller/embeddednodecustomdata.go | 10 ++++++++-- aks-node-controller/hotfix.go | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/aks-node-controller/embeddednodecustomdata.go b/aks-node-controller/embeddednodecustomdata.go index c26b8d8134a..79dab40363e 100644 --- a/aks-node-controller/embeddednodecustomdata.go +++ b/aks-node-controller/embeddednodecustomdata.go @@ -11,6 +11,12 @@ import ( const defaultOSReleasePath = "/etc/os-release" +// os-release ID values that appear in more than one classification path. +const ( + osReleaseIDAzureContainerLinux = "azurecontainerlinux" + osReleaseIDFlatcar = "flatcar" +) + type nodeCustomDataPlatform string const ( @@ -57,11 +63,11 @@ func classifyNodeCustomDataPlatform(osReleasePath string) (nodeCustomDataPlatfor switch { case variant == "osguard": return nodeCustomDataPlatformAzlOSGuard, nil - case variant == "azurecontainerlinux", id == "azurecontainerlinux": + case variant == osReleaseIDAzureContainerLinux, id == osReleaseIDAzureContainerLinux: return nodeCustomDataPlatformACL, nil case id == "ubuntu": return nodeCustomDataPlatformUbuntu, nil - case id == "flatcar": + case id == osReleaseIDFlatcar: return nodeCustomDataPlatformFlatcar, nil case id == "mariner", id == "azurelinux": return nodeCustomDataPlatformMariner, nil diff --git a/aks-node-controller/hotfix.go b/aks-node-controller/hotfix.go index 3fae8ed4ed0..616e607c133 100644 --- a/aks-node-controller/hotfix.go +++ b/aks-node-controller/hotfix.go @@ -273,7 +273,7 @@ func (a *App) detectPackageManager() (packageManager, error) { return pkgMgrApt, nil case "azurelinux", "mariner": return preferredRpmManager(), nil - case "azurecontainerlinux", "flatcar": + case osReleaseIDAzureContainerLinux, osReleaseIDFlatcar: // ACL and Flatcar are image-based/immutable distros with no apt/dnf/tdnf. // ANC self-update via a PMC package is intentionally unsupported there; // they only ever run the ANC binary baked into the VHD. From 927d2a354102b3e7d1e5de28bd04c6f9c05b066c Mon Sep 17 00:00:00 2001 From: Devin Wong Date: Fri, 4 Sep 2026 11:28:30 -0700 Subject: [PATCH 09/26] test(anc): cover hotfix binary selection in launcher spec The launcher chooses between the VHD-baked binary and the staged ${BIN_PATH}-hotfix binary before running provision. That branch decides whether the embedded script payload is ever applied, but no existing test created ${BIN_PATH}-hotfix, so the selection branch had no coverage. Add three cases: - staged hotfix binary present and executable runs provision - non-executable staged path falls back to the VHD-baked binary - the full seam: the baked binary stages the replacement while handling download-hotfix, then provision runs on the staged binary Verified by mutation: forcing the selection branch to false fails the first and third cases. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4ae1dd11-ec06-4c26-bfc8-d1ef1f89fcf9 --- .../aks_node_controller_launcher_spec.sh | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/spec/parts/linux/cloud-init/artifacts/aks_node_controller_launcher_spec.sh b/spec/parts/linux/cloud-init/artifacts/aks_node_controller_launcher_spec.sh index 6b1334dd123..4b79fb772f4 100644 --- a/spec/parts/linux/cloud-init/artifacts/aks_node_controller_launcher_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/aks_node_controller_launcher_spec.sh @@ -72,6 +72,38 @@ EOF chmod +x "$BIN_PATH" } + # Stands in for the binary download-hotfix stages at "${BIN_PATH}-hotfix". It records to a + # separate calls log so tests can prove which of the two binaries actually ran provision. + create_staged_hotfix_binary() { + cat >"${BIN_PATH}-hotfix" <<'EOF' +#!/bin/sh +printf '%s\n' "$1" >>"${TEST_DIR}/hotfix_calls" +exit 0 +EOF + chmod +x "${BIN_PATH}-hotfix" + } + + # Mirrors the real chain: the VHD-baked binary itself stages the hotfix binary while handling + # download-hotfix, so binary selection observes a file that did not exist when the wrapper started. + create_staging_aks_node_controller() { + cat >"${TEST_DIR}/hotfix-template" <<'EOF' +#!/bin/sh +printf '%s\n' "$1" >>"${TEST_DIR}/hotfix_calls" +exit 0 +EOF + + cat >"$BIN_PATH" <<'EOF' +#!/bin/sh +printf '%s\n' "$1" >>"${TEST_DIR}/calls" +if [ "$1" = "download-hotfix" ]; then + cp "${TEST_DIR}/hotfix-template" "${BIN_PATH}-hotfix" + chmod +x "${BIN_PATH}-hotfix" +fi +exit 0 +EOF + chmod +x "$BIN_PATH" + } + BeforeEach setup_wrapper_test AfterEach cleanup_wrapper_test @@ -279,4 +311,53 @@ EOF The variable firstCall should eq "check-hotfix" The variable lastCall should eq "provision" End + + # Binary selection. The hotfix binary carries the embedded script payload, so "which binary runs + # provision" decides whether that payload is ever applied. These cover the branch directly. + It 'runs the staged hotfix binary for provision when one is present' + touch "$CONFIG_PATH" + create_recording_aks_node_controller + create_staged_hotfix_binary + + When run bash "$SCRIPT" + The status should be success + The output should include "Using hotfix binary: ${BIN_PATH}-hotfix" + hotfixCall=$(tail -n 1 "${TEST_DIR}/hotfix_calls") + The variable hotfixCall should eq "provision" + # The VHD-baked binary must not have been invoked at all on this path. + The path "${TEST_DIR}/calls" should not be exist + End + + It 'falls back to the VHD-baked binary when the staged hotfix path is not executable' + touch "$CONFIG_PATH" + create_recording_aks_node_controller + create_staged_hotfix_binary + chmod -x "${BIN_PATH}-hotfix" + + When run bash "$SCRIPT" + The status should be success + The output should include "Using VHD-baked binary: ${BIN_PATH}" + lastCall=$(tail -n 1 "${TEST_DIR}/calls") + The variable lastCall should eq "provision" + The path "${TEST_DIR}/hotfix_calls" should not be exist + End + + # The full production seam for a version-only hotfix pointer: the baked binary handles + # download-hotfix and stages the replacement, then provision runs on the STAGED binary. That + # handoff is what lets the staged binary apply its compiled-in script payload; no scripts_version + # is involved. Previously uncovered, because no test created "${BIN_PATH}-hotfix". + It 'provisions with the binary staged by download-hotfix' + touch "$CONFIG_PATH" "$HOTFIX_JSON" + create_staging_aks_node_controller + + When run bash "$SCRIPT" + The status should be success + The output should include "ANC download-hotfix completed" + The output should include "Using hotfix binary" + bakedCalls=$(cat "${TEST_DIR}/calls") + hotfixCall=$(tail -n 1 "${TEST_DIR}/hotfix_calls") + # The baked binary only downloads; it must never be the one that provisions. + The variable bakedCalls should eq "download-hotfix" + The variable hotfixCall should eq "provision" + End End From 96e31d6b507e6b86fd2fbc72041fd1074c89627a Mon Sep 17 00:00:00 2001 From: Devin Wong Date: Fri, 4 Sep 2026 18:45:18 -0700 Subject: [PATCH 10/26] fix(anc): don't report unreachable LPS with no cold-start pointer as an error When check-hotfix cannot reach the LPS and the node config carries no cold-start hotfixes map, there is nothing to stage, so the existing on-disk pointer is left intact and provisioning continues normally via the fail-open path. This is the expected state for a node seeded without an injected map, but it was reported as outcomeFailed, which helpersEventLevel maps to EventLevelError -- emitting an error-level CheckHotfix guest-agent event during a completely healthy provision. Introduce a distinct benign outcome, noColdStartPointer, so the "LPS was unreachable" signal is still preserved for diagnosis (along with the wrapped fetch error in the telemetry message) without misreporting a successful fail-open as a failure. Genuine failures -- parse errors, write errors, failed cold-start reads and authoritative non-benign 4xx rejections -- continue to report outcomeFailed at error level. Observed on a standalone validation node: the LPS is unreachable there, so every healthy node emitted an error-level event. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4ae1dd11-ec06-4c26-bfc8-d1ef1f89fcf9 --- aks-node-controller/checkhotfix.go | 12 +++++- aks-node-controller/checkhotfix_test.go | 53 +++++++++++++++++++++---- 2 files changed, 56 insertions(+), 9 deletions(-) diff --git a/aks-node-controller/checkhotfix.go b/aks-node-controller/checkhotfix.go index 0820b76a5b2..6cca37893b9 100644 --- a/aks-node-controller/checkhotfix.go +++ b/aks-node-controller/checkhotfix.go @@ -103,6 +103,13 @@ const ( outcomeNoHotfixAvailable checkHotfixOutcome = "noHotfixAvailable" // outcomeCustomDataFallback: LPS read failed; the embedded customdata pointer was used. outcomeCustomDataFallback checkHotfixOutcome = "customDataFallback" + // outcomeNoColdStartPointer: the LPS could not be reached and the node config carried no + // cold-start hotfixes map, so there was nothing to stage. This is benign and expected on a + // node whose config was seeded without an injected map: download-hotfix simply keeps the + // existing on-disk pointer (the single-version one cloud-init wrote). Nothing failed, so it + // must not be reported at error level; the wrapped fetch error is still carried in the + // telemetry message to preserve why the LPS was unreachable. + outcomeNoColdStartPointer checkHotfixOutcome = "noColdStartPointer" // outcomeFailed: everything failed; nothing was staged. Provisioning still proceeds (exit 0). outcomeFailed checkHotfixOutcome = "failed" ) @@ -253,7 +260,10 @@ func (a *App) handleFetchError(hotfixPath string, fetchErr error) (checkHotfixOu return outcomeFailed, fmt.Errorf("LPS fetch failed (%w) and cold-start fallback failed: %w", fetchErr, coldErr) } if !ok { - return outcomeFailed, fmt.Errorf("LPS fetch failed and no cold-start pointer present: %w", fetchErr) + // Benign: no map was injected into the node config, so there is nothing to stage and + // the existing on-disk pointer stays intact. Report a non-error outcome while keeping + // the fetch error for diagnosis of why the LPS was unreachable. + return outcomeNoColdStartPointer, fmt.Errorf("LPS fetch failed and no cold-start pointer present: %w", fetchErr) } if err := writeHotfixConfig(hotfixPath, cfg); err != nil { return outcomeFailed, fmt.Errorf("writing cold-start hotfix config: %w", err) diff --git a/aks-node-controller/checkhotfix_test.go b/aks-node-controller/checkhotfix_test.go index ace49a87f81..25909bd7604 100644 --- a/aks-node-controller/checkhotfix_test.go +++ b/aks-node-controller/checkhotfix_test.go @@ -11,6 +11,7 @@ import ( "runtime" "testing" + "github.com/Azure/agentbaker/aks-node-controller/helpers" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/grpc/codes" @@ -249,14 +250,18 @@ func TestCheckHotfix_LPSUnavailableIsBenign(t *testing.T) { } } -func TestCheckHotfix_FetchErrorFailsOpenWithoutFallback(t *testing.T) { +// TestCheckHotfix_FetchErrorNoFallbackStagesNothing covers an unreachable LPS with no +// cold-start pointer available. Nothing may be staged, and the outcome is the benign +// noColdStartPointer rather than a failure: fail-open worked exactly as designed and the +// node still provisions from its existing on-disk pointer. +func TestCheckHotfix_FetchErrorNoFallbackStagesNothing(t *testing.T) { tt := NewTestApp(t, TestAppConfig{}) path := filepath.Join(t.TempDir(), "hotfix.json") tt.App.hotfixVersionPath = path // No node config -> no cold-start fallback available. tt.App.nodeConfigPath = filepath.Join(t.TempDir(), "nonexistent-config.json") - // Transport-level failures (not benign 401/403/404) with no fallback -> failed. + // Transport-level failures (not benign 401/403/404) with no fallback -> benign no-op. cases := map[string]error{ "timeout": context.DeadlineExceeded, "connection err": errors.New("dial tcp: connection refused"), @@ -268,8 +273,9 @@ func TestCheckHotfix_FetchErrorFailsOpenWithoutFallback(t *testing.T) { return nil, fetchErr } outcome, err := tt.App.checkHotfix(context.Background()) - assert.Equal(t, outcomeFailed, outcome) - assert.Error(t, err) + assert.Equal(t, outcomeNoColdStartPointer, outcome) + assert.Error(t, err, "the underlying fetch error is still reported for diagnosis") + assert.Equal(t, helpers.EventLevelInformational, helpersEventLevel(outcome)) // Nothing should be staged. _, statErr := os.Stat(path) assert.True(t, os.IsNotExist(statErr)) @@ -319,7 +325,13 @@ func TestCheckHotfix_ColdStartFallback(t *testing.T) { assert.Equal(t, map[string]string{"202604.01": "202604.01.2"}, cfg.Hotfixes) } -func TestCheckHotfix_ColdStartNoPointerFails(t *testing.T) { +// TestCheckHotfix_ColdStartNoPointerIsBenign verifies that an unreachable LPS combined with a +// node config carrying no cold-start hotfixes map is treated as benign, not as a failure. +// Nothing is staged and the existing on-disk pointer is left untouched, so provisioning is +// unaffected; reporting this at error level would emit misleading error telemetry on every +// healthy node whose config was seeded without an injected map. The fetch error is still +// returned so the reason the LPS was unreachable survives into the telemetry message. +func TestCheckHotfix_ColdStartNoPointerIsBenign(t *testing.T) { tt := NewTestApp(t, TestAppConfig{}) path := filepath.Join(t.TempDir(), "hotfix.json") tt.App.hotfixVersionPath = path @@ -332,8 +344,10 @@ func TestCheckHotfix_ColdStartNoPointerFails(t *testing.T) { } outcome, err := tt.App.checkHotfix(context.Background()) - assert.Equal(t, outcomeFailed, outcome) + assert.Equal(t, outcomeNoColdStartPointer, outcome) assert.Error(t, err) + assert.Equal(t, helpers.EventLevelInformational, helpersEventLevel(outcome), + "a benign no-op must not be reported at error level") _, statErr := os.Stat(path) assert.True(t, os.IsNotExist(statErr)) } @@ -410,12 +424,12 @@ func TestRunCheckHotfixCommand_AlwaysFailOpen(t *testing.T) { assert.Contains(t, events[0].Message, string(outcomeLPSRead)) }) - t.Run("failure path emits error event but still exits 0", func(t *testing.T) { + t.Run("authoritative client error emits error event but still exits 0", func(t *testing.T) { tt := NewTestApp(t, TestAppConfig{}) tt.App.hotfixVersionPath = filepath.Join(t.TempDir(), "hotfix.json") tt.App.nodeConfigPath = filepath.Join(t.TempDir(), "nonexistent.json") tt.App.checkHotfixFetcher = func(context.Context) ([]byte, error) { - return nil, errors.New("LPS returned status 500") + return nil, &lpsGRPCStatusError{code: codes.ResourceExhausted, fallbackAllowed: false} } err := tt.App.runCheckHotfixCommand(context.Background()) @@ -428,6 +442,29 @@ func TestRunCheckHotfixCommand_AlwaysFailOpen(t *testing.T) { assert.Contains(t, events[0].Message, string(outcomeFailed)) }) + // An unreachable LPS on a node with no injected cold-start map is the common healthy case + // (it is exactly what a first-boot node sees when the LPS is briefly unavailable). It must + // emit an Informational event, not an Error, while still recording why the fetch failed. + t.Run("unreachable LPS without a cold-start pointer emits an informational event", func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + tt.App.hotfixVersionPath = filepath.Join(t.TempDir(), "hotfix.json") + tt.App.nodeConfigPath = filepath.Join(t.TempDir(), "nonexistent.json") + tt.App.checkHotfixFetcher = func(context.Context) ([]byte, error) { + return nil, errors.New("LPS returned status 500") + } + + err := tt.App.runCheckHotfixCommand(context.Background()) + require.NoError(t, err) + + events := tt.eventLogger.Events() + require.Len(t, events, 1) + assert.Equal(t, "AKS.AKSNodeController.CheckHotfix", events[0].TaskName) + assert.Equal(t, "Informational", events[0].EventLevel) + assert.Contains(t, events[0].Message, string(outcomeNoColdStartPointer)) + assert.Contains(t, events[0].Message, "LPS returned status 500", + "the fetch error must survive into the message for diagnosis") + }) + t.Run("cli wiring returns exit code 0 even on fetch failure", func(t *testing.T) { tt := NewTestApp(t, TestAppConfig{}) tt.App.hotfixVersionPath = filepath.Join(t.TempDir(), "hotfix.json") From 0d54b550782fe9bbdf0900218f55814b5533041d Mon Sep 17 00:00:00 2001 From: Devin Wong Date: Tue, 8 Sep 2026 15:36:47 -0700 Subject: [PATCH 11/26] fix(anc): align hotfix completion logs with outcome severity Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4ae1dd11-ec06-4c26-bfc8-d1ef1f89fcf9 --- aks-node-controller/checkhotfix.go | 4 +++ aks-node-controller/checkhotfix_test.go | 37 +++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/aks-node-controller/checkhotfix.go b/aks-node-controller/checkhotfix.go index 6cca37893b9..15e9a2da681 100644 --- a/aks-node-controller/checkhotfix.go +++ b/aks-node-controller/checkhotfix.go @@ -171,7 +171,11 @@ func (a *App) runCheckHotfixCommand(ctx context.Context) (err error) { message := fmt.Sprintf("check-hotfix outcome=%s", outcome) if err != nil { message = fmt.Sprintf("%s error=%s", message, err.Error()) + } + if level == helpers.EventLevelError { slog.Warn("check-hotfix completed with error (fail-open)", "outcome", outcome, "error", err) + } else if err != nil { + slog.Info("check-hotfix completed (fail-open)", "outcome", outcome, "reason", err) } else { slog.Info("check-hotfix completed", "outcome", outcome) } diff --git a/aks-node-controller/checkhotfix_test.go b/aks-node-controller/checkhotfix_test.go index 25909bd7604..833f5383162 100644 --- a/aks-node-controller/checkhotfix_test.go +++ b/aks-node-controller/checkhotfix_test.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "log/slog" "os" "path/filepath" "runtime" @@ -404,6 +405,7 @@ func TestCheckHotfix_FallbackOnlyForUnreachableLPS(t *testing.T) { // (exit 0) and emits telemetry, regardless of the underlying outcome. func TestRunCheckHotfixCommand_AlwaysFailOpen(t *testing.T) { t.Run("success path emits informational event and exits 0", func(t *testing.T) { + logCap := installLogCapturer(t) origVersion := Version Version = "202604.01.0" defer func() { Version = origVersion }() @@ -422,9 +424,15 @@ func TestRunCheckHotfixCommand_AlwaysFailOpen(t *testing.T) { assert.Equal(t, "AKS.AKSNodeController.CheckHotfix", events[0].TaskName) assert.Equal(t, "Informational", events[0].EventLevel) assert.Contains(t, events[0].Message, string(outcomeLPSRead)) + assert.Contains(t, logCap.getRecords(), logRecord{ + Level: slog.LevelInfo, + Message: "check-hotfix completed", + Attrs: map[string]string{"outcome": string(outcomeLPSRead)}, + }) }) t.Run("authoritative client error emits error event but still exits 0", func(t *testing.T) { + logCap := installLogCapturer(t) tt := NewTestApp(t, TestAppConfig{}) tt.App.hotfixVersionPath = filepath.Join(t.TempDir(), "hotfix.json") tt.App.nodeConfigPath = filepath.Join(t.TempDir(), "nonexistent.json") @@ -440,12 +448,23 @@ func TestRunCheckHotfixCommand_AlwaysFailOpen(t *testing.T) { assert.Equal(t, "AKS.AKSNodeController.CheckHotfix", events[0].TaskName) assert.Equal(t, "Error", events[0].EventLevel) assert.Contains(t, events[0].Message, string(outcomeFailed)) + var completions []logRecord + for _, record := range logCap.getRecords() { + if record.Attrs["outcome"] == string(outcomeFailed) { + completions = append(completions, record) + } + } + require.Len(t, completions, 1) + assert.Equal(t, slog.LevelWarn, completions[0].Level) + assert.Equal(t, "check-hotfix completed with error (fail-open)", completions[0].Message) + assert.NotEmpty(t, completions[0].Attrs["error"]) }) // An unreachable LPS on a node with no injected cold-start map is the common healthy case // (it is exactly what a first-boot node sees when the LPS is briefly unavailable). It must // emit an Informational event, not an Error, while still recording why the fetch failed. t.Run("unreachable LPS without a cold-start pointer emits an informational event", func(t *testing.T) { + logCap := installLogCapturer(t) tt := NewTestApp(t, TestAppConfig{}) tt.App.hotfixVersionPath = filepath.Join(t.TempDir(), "hotfix.json") tt.App.nodeConfigPath = filepath.Join(t.TempDir(), "nonexistent.json") @@ -463,6 +482,24 @@ func TestRunCheckHotfixCommand_AlwaysFailOpen(t *testing.T) { assert.Contains(t, events[0].Message, string(outcomeNoColdStartPointer)) assert.Contains(t, events[0].Message, "LPS returned status 500", "the fetch error must survive into the message for diagnosis") + + records := logCap.getRecords() + assert.Contains(t, records, logRecord{ + Level: slog.LevelWarn, + Message: "failed to reach LPS, attempting cold-start fallback", + Attrs: map[string]string{"error": "LPS returned status 500"}, + }) + var completions []logRecord + for _, record := range records { + if record.Attrs["outcome"] == string(outcomeNoColdStartPointer) { + completions = append(completions, record) + } + } + require.Len(t, completions, 1) + assert.Equal(t, slog.LevelInfo, completions[0].Level) + assert.Equal(t, "check-hotfix completed (fail-open)", completions[0].Message) + assert.Contains(t, completions[0].Attrs["reason"], "LPS returned status 500") + assert.NotContains(t, completions[0].Attrs, "error") }) t.Run("cli wiring returns exit code 0 even on fetch failure", func(t *testing.T) { From bd3cfa5c7abee29689e4b521ea516312cdeb0171 Mon Sep 17 00:00:00 2001 From: Devin Wong Date: Tue, 8 Sep 2026 15:36:47 -0700 Subject: [PATCH 12/26] fix(anc): limit embedded script hotfixes to Ubuntu and Mariner Retain only the two inactive empty templates and skip OS Guard, ACL, and Flatcar during generation and embedded application. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4ae1dd11-ec06-4c26-bfc8-d1ef1f89fcf9 --- aks-node-controller/README.md | 12 +- aks-node-controller/embeddednodecustomdata.go | 28 +++-- .../embeddednodecustomdata_test.go | 108 +++++++++++++----- .../generated/rendered_nodecustomdata_acl.yml | 2 - .../rendered_nodecustomdata_azlosguard.yml | 2 - .../rendered_nodecustomdata_flatcar.yml | 2 - hotfix/hotfix_generate.py | 18 +-- hotfix/hotfix_generate_test.py | 30 +++-- hotfix/render-nodecustomdata/main.go | 3 - 9 files changed, 125 insertions(+), 80 deletions(-) delete mode 100644 aks-node-controller/scripthotfix/generated/rendered_nodecustomdata_acl.yml delete mode 100644 aks-node-controller/scripthotfix/generated/rendered_nodecustomdata_azlosguard.yml delete mode 100644 aks-node-controller/scripthotfix/generated/rendered_nodecustomdata_flatcar.yml diff --git a/aks-node-controller/README.md b/aks-node-controller/README.md index 16713ca5c7e..444c8971780 100644 --- a/aks-node-controller/README.md +++ b/aks-node-controller/README.md @@ -163,9 +163,15 @@ The embedded nodecustomdata coordinator distinguishes these script hotfixes from updates to the ANC binary itself. The generated files live under `aks-node-controller/scripthotfix/generated/` as `rendered_nodecustomdata_.yml`. The generator selects only changed -hotfixable entries from `nodecustomdata.yml`, then renders Ubuntu, Mariner/Azure -Linux, ACL, OS Guard, and Flatcar variants through AgentBaker's production -Go-template functions. +hotfixable entries from `nodecustomdata.yml`, then renders only Ubuntu and +Mariner/standard Azure Linux variants through AgentBaker's production Go-template +functions. OS Guard, ACL, and Flatcar are explicitly skipped during embedded +application, including variants that share the `azurelinux` OS ID. Their +distro-specific source changes do not trigger payload generation. + +The repository keeps two empty YAML templates (`write_files: []`) and an +`active=false` marker until a script hotfix is generated. Generation populates +those two payloads and sets `active=true`. When a PR has no new script hotfix, generation leaves the existing rendered payload unchanged. The active ANC version pointer is likewise retained until it diff --git a/aks-node-controller/embeddednodecustomdata.go b/aks-node-controller/embeddednodecustomdata.go index 79dab40363e..0103fc2bbcc 100644 --- a/aks-node-controller/embeddednodecustomdata.go +++ b/aks-node-controller/embeddednodecustomdata.go @@ -4,6 +4,7 @@ import ( "embed" "fmt" "io/fs" + "log/slog" "os" "path/filepath" "strings" @@ -20,11 +21,9 @@ const ( type nodeCustomDataPlatform string const ( - nodeCustomDataPlatformUbuntu nodeCustomDataPlatform = "ubuntu" - nodeCustomDataPlatformMariner nodeCustomDataPlatform = "mariner" - nodeCustomDataPlatformACL nodeCustomDataPlatform = "acl" - nodeCustomDataPlatformAzlOSGuard nodeCustomDataPlatform = "azlosguard" - nodeCustomDataPlatformFlatcar nodeCustomDataPlatform = "flatcar" + nodeCustomDataPlatformUbuntu nodeCustomDataPlatform = "ubuntu" + nodeCustomDataPlatformMariner nodeCustomDataPlatform = "mariner" + nodeCustomDataPlatformUnsupported nodeCustomDataPlatform = "unsupported" ) //go:embed scripthotfix/generated @@ -48,6 +47,10 @@ func applyEmbeddedNodeCustomDataIfActive(osReleasePath string) (nodeCustomDataAp if err != nil { return nodeCustomDataApplyResult{}, err } + if platform == nodeCustomDataPlatformUnsupported { + slog.Info("embedded script hotfix is not supported on this OS, skipping", "osReleasePath", osReleasePath) + return nodeCustomDataApplyResult{}, nil + } return applyEmbeddedNodeCustomDataFS(generatedNodeCustomData, platform) } @@ -61,14 +64,12 @@ func classifyNodeCustomDataPlatform(osReleasePath string) (nodeCustomDataPlatfor variant := strings.ToLower(values["VARIANT_ID"]) switch { - case variant == "osguard": - return nodeCustomDataPlatformAzlOSGuard, nil - case variant == osReleaseIDAzureContainerLinux, id == osReleaseIDAzureContainerLinux: - return nodeCustomDataPlatformACL, nil + // Exclude immutable variants before matching their shared Azure Linux ID. + case variant == "osguard", variant == osReleaseIDAzureContainerLinux, + id == osReleaseIDAzureContainerLinux, id == osReleaseIDFlatcar: + return nodeCustomDataPlatformUnsupported, nil case id == "ubuntu": return nodeCustomDataPlatformUbuntu, nil - case id == osReleaseIDFlatcar: - return nodeCustomDataPlatformFlatcar, nil case id == "mariner", id == "azurelinux": return nodeCustomDataPlatformMariner, nil case id == "": @@ -123,10 +124,7 @@ func applyEmbeddedNodeCustomDataFS( func isConcreteNodeCustomDataPlatform(platform nodeCustomDataPlatform) bool { switch platform { case nodeCustomDataPlatformUbuntu, - nodeCustomDataPlatformMariner, - nodeCustomDataPlatformACL, - nodeCustomDataPlatformAzlOSGuard, - nodeCustomDataPlatformFlatcar: + nodeCustomDataPlatformMariner: return true default: return false diff --git a/aks-node-controller/embeddednodecustomdata_test.go b/aks-node-controller/embeddednodecustomdata_test.go index e50392d46e5..537c02fcd87 100644 --- a/aks-node-controller/embeddednodecustomdata_test.go +++ b/aks-node-controller/embeddednodecustomdata_test.go @@ -28,15 +28,15 @@ func TestClassifyNodeCustomDataPlatform(t *testing.T) { { name: "OS Guard variant wins over Azure Linux ID", release: "ID=azurelinux\nVARIANT_ID=osguard\n", - expected: nodeCustomDataPlatformAzlOSGuard, + expected: nodeCustomDataPlatformUnsupported, }, { name: "ACL variant wins over Azure Linux ID", release: "ID=azurelinux\nVARIANT_ID=azurecontainerlinux\n", - expected: nodeCustomDataPlatformACL, + expected: nodeCustomDataPlatformUnsupported, }, - {name: "ACL dedicated ID", release: "ID=azurecontainerlinux\n", expected: nodeCustomDataPlatformACL}, - {name: "Flatcar", release: "ID=flatcar\n", expected: nodeCustomDataPlatformFlatcar}, + {name: "ACL dedicated ID", release: "ID=azurecontainerlinux\n", expected: nodeCustomDataPlatformUnsupported}, + {name: "Flatcar", release: "ID=flatcar\n", expected: nodeCustomDataPlatformUnsupported}, } for _, test := range tests { @@ -76,40 +76,90 @@ func TestApplyEmbeddedNodeCustomDataIfActiveSkipsInactivePayload(t *testing.T) { assert.Equal(t, nodeCustomDataApplyResult{}, result) } -func TestApplyEmbeddedNodeCustomDataIfActiveSelectsPlatformPayload(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("Windows rename cannot atomically replace an existing destination") - } - - directory := t.TempDir() - destination := filepath.Join(directory, "provision.sh") - require.NoError(t, os.WriteFile(destination, []byte("old"), 0o600)) - releasePath := filepath.Join(directory, "os-release") - require.NoError(t, os.WriteFile(releasePath, []byte("ID=ubuntu\n"), 0o600)) - payload := []byte("#!/bin/sh\necho fixed\n") - +func TestApplyEmbeddedNodeCustomDataIfActiveSkipsUnsupportedPlatforms(t *testing.T) { original := generatedNodeCustomData generatedNodeCustomData = fstest.MapFS{ "scripthotfix/generated/active": &fstest.MapFile{Data: []byte("true\n")}, - embeddedRenderedPath(nodeCustomDataPlatformUbuntu): &fstest.MapFile{Data: marshalNodeCustomData(t, []nodeCustomDataWriteFile{{ - Path: destination, - Permissions: "0744", - Encoding: encodingBase64, - Owner: "root", - Content: base64.StdEncoding.EncodeToString(payload), - }})}, + // A variant accidentally classified as Mariner must fail, not silently pass. + embeddedRenderedPath(nodeCustomDataPlatformMariner): &fstest.MapFile{Data: []byte("invalid: [")}, } t.Cleanup(func() { generatedNodeCustomData = original }) - result, err := applyEmbeddedNodeCustomDataIfActive(releasePath) + for _, release := range []string{ + "ID=azurelinux\nVARIANT_ID=osguard\n", + "ID=azurelinux\nVARIANT_ID=azurecontainerlinux\n", + "ID=azurecontainerlinux\n", + "ID=flatcar\n", + "ID=\"AZURELINUX\"\nVARIANT_ID=\"OSGUARD\"\n", + } { + t.Run(release, func(t *testing.T) { + releasePath := filepath.Join(t.TempDir(), "os-release") + require.NoError(t, os.WriteFile(releasePath, []byte(release), 0o600)) - require.NoError(t, err) - assert.Equal(t, nodeCustomDataApplyResult{Applied: 1}, result) - actual, err := os.ReadFile(destination) - require.NoError(t, err) - assert.Equal(t, payload, actual) + result, err := applyEmbeddedNodeCustomDataIfActive(releasePath) + + require.NoError(t, err) + assert.Equal(t, nodeCustomDataApplyResult{}, result) + }) + } +} + +func TestApplyEmbeddedNodeCustomDataFSRejectsUnsupportedPlatforms(t *testing.T) { + for _, platform := range []nodeCustomDataPlatform{"acl", "azlosguard", "flatcar", nodeCustomDataPlatformUnsupported} { + t.Run(string(platform), func(t *testing.T) { + _, err := applyEmbeddedNodeCustomDataFS(fstest.MapFS{}, platform) + require.ErrorContains(t, err, "unsupported concrete platform") + }) + } +} + +func TestApplyEmbeddedNodeCustomDataIfActiveSelectsPlatformPayload(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Windows rename cannot atomically replace an existing destination") + } + + for _, test := range []struct { + id string + platform nodeCustomDataPlatform + }{ + {id: "ubuntu", platform: nodeCustomDataPlatformUbuntu}, + {id: "mariner", platform: nodeCustomDataPlatformMariner}, + {id: "azurelinux", platform: nodeCustomDataPlatformMariner}, + } { + t.Run(test.id, func(t *testing.T) { + directory := t.TempDir() + destination := filepath.Join(directory, "provision.sh") + require.NoError(t, os.WriteFile(destination, []byte("old"), 0o600)) + releasePath := filepath.Join(directory, "os-release") + require.NoError(t, os.WriteFile(releasePath, []byte("ID="+test.id+"\n"), 0o600)) + payload := []byte("#!/bin/sh\necho fixed\n") + + original := generatedNodeCustomData + generatedNodeCustomData = fstest.MapFS{ + "scripthotfix/generated/active": &fstest.MapFile{Data: []byte("true\n")}, + embeddedRenderedPath(test.platform): &fstest.MapFile{Data: marshalNodeCustomData(t, []nodeCustomDataWriteFile{{ + Path: destination, + Permissions: "0744", + Encoding: encodingBase64, + Owner: "root", + Content: base64.StdEncoding.EncodeToString(payload), + }})}, + } + t.Cleanup(func() { + generatedNodeCustomData = original + }) + + result, err := applyEmbeddedNodeCustomDataIfActive(releasePath) + + require.NoError(t, err) + assert.Equal(t, nodeCustomDataApplyResult{Applied: 1}, result) + actual, err := os.ReadFile(destination) + require.NoError(t, err) + assert.Equal(t, payload, actual) + }) + } } func TestApplyEmbeddedNodeCustomDataIsReplaceOnlyAndIdempotent(t *testing.T) { diff --git a/aks-node-controller/scripthotfix/generated/rendered_nodecustomdata_acl.yml b/aks-node-controller/scripthotfix/generated/rendered_nodecustomdata_acl.yml deleted file mode 100644 index 7028abd713c..00000000000 --- a/aks-node-controller/scripthotfix/generated/rendered_nodecustomdata_acl.yml +++ /dev/null @@ -1,2 +0,0 @@ -#cloud-config -write_files: [] diff --git a/aks-node-controller/scripthotfix/generated/rendered_nodecustomdata_azlosguard.yml b/aks-node-controller/scripthotfix/generated/rendered_nodecustomdata_azlosguard.yml deleted file mode 100644 index 7028abd713c..00000000000 --- a/aks-node-controller/scripthotfix/generated/rendered_nodecustomdata_azlosguard.yml +++ /dev/null @@ -1,2 +0,0 @@ -#cloud-config -write_files: [] diff --git a/aks-node-controller/scripthotfix/generated/rendered_nodecustomdata_flatcar.yml b/aks-node-controller/scripthotfix/generated/rendered_nodecustomdata_flatcar.yml deleted file mode 100644 index 7028abd713c..00000000000 --- a/aks-node-controller/scripthotfix/generated/rendered_nodecustomdata_flatcar.yml +++ /dev/null @@ -1,2 +0,0 @@ -#cloud-config -write_files: [] diff --git a/hotfix/hotfix_generate.py b/hotfix/hotfix_generate.py index e95fcacc896..3bf4d1dccd4 100644 --- a/hotfix/hotfix_generate.py +++ b/hotfix/hotfix_generate.py @@ -11,7 +11,7 @@ 2. Detects which CSE provisioning scripts differ from the immutable VHD baseline (the release tag the VHD was built from, derived from linux_sig_version.json), selects their write_files entries from parts/linux/cloud-init/nodecustomdata.yml, - and renders self-contained ANC payloads for each Linux platform with AgentBaker's + and renders self-contained ANC payloads for Ubuntu and Mariner with AgentBaker's canonical Go-template renderer. Diffing against the frozen baseline (rather than the moving base branch) keeps every generated payload cumulative: a later hotfix re-renders all scripts changed since the VHD, so it never silently drops an @@ -55,17 +55,11 @@ # CSE helpers — distro variants (all map to the same conditional block) "ubuntu/cse_helpers_ubuntu.sh": "provisionSourceUbuntu", "mariner/cse_helpers_mariner.sh": "provisionSourceMariner", - "azlosguard/cse_helpers_osguard.sh": "provisionSourceAzlOSGuard", - "flatcar/cse_helpers_flatcar.sh": "provisionSourceFlatcar", - "acl/cse_helpers_acl.sh": "provisionSourceACL", # CSE install — base "cse_install.sh": "provisionInstalls", # CSE install — distro variants "ubuntu/cse_install_ubuntu.sh": "provisionInstallsUbuntu", "mariner/cse_install_mariner.sh": "provisionInstallsMariner", - "azlosguard/cse_install_osguard.sh": "provisionInstallsAzlOSGuard", - "flatcar/cse_install_flatcar.sh": "provisionInstallsFlatcar", - "acl/cse_install_acl.sh": "provisionInstallsACL", # CSE config "cse_config.sh": "provisionConfigs", # CSE main / start @@ -85,17 +79,12 @@ VARKEY_TO_BLOCK_GROUP = { "provisionSourceUbuntu": "helpers_distro", "provisionSourceMariner": "helpers_distro", - "provisionSourceAzlOSGuard": "helpers_distro", - "provisionSourceFlatcar": "helpers_distro", - "provisionSourceACL": "helpers_distro", "provisionInstallsUbuntu": "install_distro", "provisionInstallsMariner": "install_distro", - "provisionInstallsAzlOSGuard": "install_distro", - "provisionInstallsFlatcar": "install_distro", - "provisionInstallsACL": "install_distro", } VARKEY_TO_SOURCE = {varkey: source for source, varkey in SOURCE_TO_VARKEY.items()} +UNSUPPORTED_DISTRO_DIRS = ("acl/", "azlosguard/", "flatcar/") HOTFIXABLE_SUFFIXES = ( ".sh", @@ -246,6 +235,9 @@ def detect_changed_varkeys(base_ref, available_varkeys=None): local_path = filepath.removeprefix(f"{ARTIFACTS_DIR}/") if local_path in GENERATED_ARTIFACTS: continue + if local_path.startswith(UNSUPPORTED_DISTRO_DIRS): + print(f" Skipping unsupported embedded hotfix distro: {local_path}") + continue if local_path in SOURCE_TO_VARKEY: source_path = os.path.join(ARTIFACTS_DIR, local_path) if not os.path.isfile(source_path): diff --git a/hotfix/hotfix_generate_test.py b/hotfix/hotfix_generate_test.py index 8a34ee0f002..17c7c28db8e 100644 --- a/hotfix/hotfix_generate_test.py +++ b/hotfix/hotfix_generate_test.py @@ -113,9 +113,6 @@ def test_detect_changed_varkeys_expands_distro_group(self): for source in ( "ubuntu/cse_helpers_ubuntu.sh", "mariner/cse_helpers_mariner.sh", - "azlosguard/cse_helpers_osguard.sh", - "flatcar/cse_helpers_flatcar.sh", - "acl/cse_helpers_acl.sh", ): path = artifacts / source path.parent.mkdir(parents=True, exist_ok=True) @@ -144,7 +141,24 @@ def test_detect_changed_varkeys_expands_distro_group(self): available_varkeys=available, ) - self.assertEqual(available, selected) + self.assertEqual( + {"provisionSourceUbuntu", "provisionSourceMariner"}, selected + ) + + def test_detect_changed_varkeys_skips_unsupported_distros(self): + for distro in ("acl", "azlosguard", "flatcar"): + with self.subTest(distro=distro): + result = subprocess.CompletedProcess( + args=[], + returncode=0, + stdout=f"{hotfix_generate.ARTIFACTS_DIR}/{distro}/cse_helpers_{distro}.sh\n", + ) + with mock.patch.object( + hotfix_generate.subprocess, "run", return_value=result + ): + self.assertEqual( + set(), hotfix_generate.detect_changed_varkeys("base") + ) def test_write_rendered_payload_uses_canonical_renderer(self): with tempfile.TemporaryDirectory() as temp_dir: @@ -157,9 +171,6 @@ def render(command, check): for platform in ( "ubuntu", "mariner", - "acl", - "azlosguard", - "flatcar", ): (output_dir / f"rendered_nodecustomdata_{platform}.yml").write_text( "#cloud-config\n" @@ -183,9 +194,6 @@ def render(command, check): expected = { "ubuntu", "mariner", - "acl", - "azlosguard", - "flatcar", } actual = { path.name.removeprefix("rendered_nodecustomdata_").removesuffix(".yml") @@ -204,7 +212,7 @@ def test_write_rendered_payload_preserves_previous_hotfix_when_unchanged(self): generated = Path(temp_dir) / "generated" generated.mkdir() (generated / "active").write_text("true\n") - platforms = ("ubuntu", "mariner", "acl", "azlosguard", "flatcar") + platforms = ("ubuntu", "mariner") for platform in platforms: (generated / f"rendered_nodecustomdata_{platform}.yml").write_text( f"write_files:\n- path: /{platform}-existing\n" diff --git a/hotfix/render-nodecustomdata/main.go b/hotfix/render-nodecustomdata/main.go index c2ea11c11d5..26aa84ba586 100644 --- a/hotfix/render-nodecustomdata/main.go +++ b/hotfix/render-nodecustomdata/main.go @@ -22,9 +22,6 @@ func main() { platforms := []platform{ {name: "ubuntu", distro: datamodel.AKSUbuntuContainerd2204Gen2}, {name: "mariner", distro: datamodel.AKSAzureLinuxV3Gen2}, - {name: "acl", distro: datamodel.AKSACLGen2TL}, - {name: "azlosguard", distro: datamodel.AKSAzureLinuxV3OSGuardGen2FIPSTL}, - {name: "flatcar", distro: datamodel.AKSFlatcarGen2}, } templatePath := flag.String("template", "", "path to the hotfix nodecustomdata template") From 510d4970a4fe2f4e40cdce647f16064e30ae9b4e Mon Sep 17 00:00:00 2001 From: Devin Wong Date: Tue, 8 Sep 2026 15:48:55 -0700 Subject: [PATCH 13/26] fix(anc): resolve hotfix lint failures Use a switch for outcome logging and explicit comparisons for supported embedded platforms. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4ae1dd11-ec06-4c26-bfc8-d1ef1f89fcf9 --- aks-node-controller/checkhotfix.go | 7 ++++--- aks-node-controller/embeddednodecustomdata.go | 8 +------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/aks-node-controller/checkhotfix.go b/aks-node-controller/checkhotfix.go index 15e9a2da681..9c5f2995f53 100644 --- a/aks-node-controller/checkhotfix.go +++ b/aks-node-controller/checkhotfix.go @@ -172,11 +172,12 @@ func (a *App) runCheckHotfixCommand(ctx context.Context) (err error) { if err != nil { message = fmt.Sprintf("%s error=%s", message, err.Error()) } - if level == helpers.EventLevelError { + switch { + case level == helpers.EventLevelError: slog.Warn("check-hotfix completed with error (fail-open)", "outcome", outcome, "error", err) - } else if err != nil { + case err != nil: slog.Info("check-hotfix completed (fail-open)", "outcome", outcome, "reason", err) - } else { + default: slog.Info("check-hotfix completed", "outcome", outcome) } if a.eventLogger != nil { diff --git a/aks-node-controller/embeddednodecustomdata.go b/aks-node-controller/embeddednodecustomdata.go index 0103fc2bbcc..6b6429880f2 100644 --- a/aks-node-controller/embeddednodecustomdata.go +++ b/aks-node-controller/embeddednodecustomdata.go @@ -122,11 +122,5 @@ func applyEmbeddedNodeCustomDataFS( } func isConcreteNodeCustomDataPlatform(platform nodeCustomDataPlatform) bool { - switch platform { - case nodeCustomDataPlatformUbuntu, - nodeCustomDataPlatformMariner: - return true - default: - return false - } + return platform == nodeCustomDataPlatformUbuntu || platform == nodeCustomDataPlatformMariner } From d3c361b384df079788ec4c57620e7a656b30fe05 Mon Sep 17 00:00:00 2001 From: Devin Wong Date: Wed, 9 Sep 2026 11:15:14 -0700 Subject: [PATCH 14/26] fix: address embedded hotfix review feedback Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- aks-node-controller/hotfix.go | 11 +++++++++++ aks-node-controller/hotfix_test.go | 16 ++++++++++++++++ pkg/agent/baker.go | 6 +++++- pkg/agent/nodecustomdata_render_test.go | 9 +++++++++ 4 files changed, 41 insertions(+), 1 deletion(-) diff --git a/aks-node-controller/hotfix.go b/aks-node-controller/hotfix.go index 616e607c133..42b83030fd8 100644 --- a/aks-node-controller/hotfix.go +++ b/aks-node-controller/hotfix.go @@ -222,6 +222,7 @@ func readHotfixConfig(path string) (*hotfixConfig, error) { type platformInfo struct { OS string // e.g. "linux", "windows" ID string // e.g. "ubuntu", "azurelinux", "mariner" + VariantID string // e.g. "azurecontainerlinux", "osguard" VersionID string // e.g. "22.04", "3.0" Arch string // e.g. "amd64", "arm64" } @@ -243,6 +244,9 @@ func (a *App) parseLinuxPlatformInfo() (platformInfo, error) { if strings.HasPrefix(line, "ID=") { info.ID = strings.ToLower(strings.Trim(strings.TrimPrefix(line, "ID="), `"`)) } + if strings.HasPrefix(line, "VARIANT_ID=") { + info.VariantID = strings.ToLower(strings.Trim(strings.TrimPrefix(line, "VARIANT_ID="), `"`)) + } if strings.HasPrefix(line, "VERSION_ID=") { info.VersionID = strings.Trim(strings.TrimPrefix(line, "VERSION_ID="), `"`) } @@ -268,6 +272,13 @@ func (a *App) detectPackageManager() (packageManager, error) { if err != nil { return "", err } + if info.ID == "azurelinux" && info.VariantID == osReleaseIDAzureContainerLinux { + return "", fmt.Errorf( + "PMC package-based ANC self-update is not supported on image-based OS %q variant %q", + info.ID, + info.VariantID, + ) + } switch info.ID { case "ubuntu": return pkgMgrApt, nil diff --git a/aks-node-controller/hotfix_test.go b/aks-node-controller/hotfix_test.go index c8278a522e4..3e775354831 100644 --- a/aks-node-controller/hotfix_test.go +++ b/aks-node-controller/hotfix_test.go @@ -187,6 +187,22 @@ func TestDetectPackageManager(t *testing.T) { } }) + t.Run("ACL azurelinux variant reports self-update unsupported", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "os-release") + require.NoError(t, os.WriteFile( + path, + []byte("ID=azurelinux\nVARIANT_ID=azurecontainerlinux\n"), + 0644, + )) + a := &App{osReleasePath: path} + + _, err := a.detectPackageManager() + + require.Error(t, err) + assert.Contains(t, err.Error(), "not supported on image-based OS") + assert.Contains(t, err.Error(), "azurecontainerlinux") + }) + t.Run("missing ID line errors", func(t *testing.T) { path := filepath.Join(t.TempDir(), "os-release") require.NoError(t, os.WriteFile(path, []byte("VERSION_ID=1\n"), 0644)) diff --git a/pkg/agent/baker.go b/pkg/agent/baker.go index 4334fe55071..db56c37de32 100644 --- a/pkg/agent/baker.go +++ b/pkg/agent/baker.go @@ -635,7 +635,11 @@ func (t *TemplateGenerator) getSingleLine(textFilename string, profile interface // RenderLinuxNodeCustomDataTemplate renders a nodecustomdata template with the // same variables and functions used by the production AgentBaker path. func RenderLinuxNodeCustomDataTemplate(templateContent []byte, config *datamodel.NodeBootstrappingConfiguration) (string, error) { - if config == nil || config.AgentPoolProfile == nil || config.ContainerService == nil || config.ContainerService.Properties == nil { + if config == nil || + config.AgentPoolProfile == nil || + config.ContainerService == nil || + config.ContainerService.Properties == nil || + config.ContainerService.Properties.OrchestratorProfile == nil { return "", fmt.Errorf("node bootstrapping configuration is incomplete") } diff --git a/pkg/agent/nodecustomdata_render_test.go b/pkg/agent/nodecustomdata_render_test.go index d96cd1f197c..e271cb4aec7 100644 --- a/pkg/agent/nodecustomdata_render_test.go +++ b/pkg/agent/nodecustomdata_render_test.go @@ -49,6 +49,15 @@ write_files: } } +func TestRenderLinuxNodeCustomDataTemplateRejectsMissingOrchestratorProfile(t *testing.T) { + config := newNodeCustomDataRenderConfig(datamodel.AKSUbuntuContainerd2204Gen2) + config.ContainerService.Properties.OrchestratorProfile = nil + + _, err := RenderLinuxNodeCustomDataTemplate([]byte("#cloud-config\nwrite_files: []\n"), config) + + require.EqualError(t, err, "node bootstrapping configuration is incomplete") +} + func newNodeCustomDataRenderConfig(distro datamodel.Distro) *datamodel.NodeBootstrappingConfiguration { profile := &datamodel.AgentPoolProfile{ Name: "hotfix-render-test", From 6690272d1153913577a3501dbe55710e7e0c6bc1 Mon Sep 17 00:00:00 2001 From: Devin Wong Date: Wed, 9 Sep 2026 11:33:54 -0700 Subject: [PATCH 15/26] fix: share Azure Linux OS identifier Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- aks-node-controller/embeddednodecustomdata.go | 3 ++- aks-node-controller/hotfix.go | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/aks-node-controller/embeddednodecustomdata.go b/aks-node-controller/embeddednodecustomdata.go index 6b6429880f2..481d5fb79a5 100644 --- a/aks-node-controller/embeddednodecustomdata.go +++ b/aks-node-controller/embeddednodecustomdata.go @@ -15,6 +15,7 @@ const defaultOSReleasePath = "/etc/os-release" // os-release ID values that appear in more than one classification path. const ( osReleaseIDAzureContainerLinux = "azurecontainerlinux" + osReleaseIDAzureLinux = "azurelinux" osReleaseIDFlatcar = "flatcar" ) @@ -70,7 +71,7 @@ func classifyNodeCustomDataPlatform(osReleasePath string) (nodeCustomDataPlatfor return nodeCustomDataPlatformUnsupported, nil case id == "ubuntu": return nodeCustomDataPlatformUbuntu, nil - case id == "mariner", id == "azurelinux": + case id == "mariner", id == osReleaseIDAzureLinux: return nodeCustomDataPlatformMariner, nil case id == "": return "", fmt.Errorf("ID is missing from %s", osReleasePath) diff --git a/aks-node-controller/hotfix.go b/aks-node-controller/hotfix.go index 42b83030fd8..353dacb0efd 100644 --- a/aks-node-controller/hotfix.go +++ b/aks-node-controller/hotfix.go @@ -272,7 +272,7 @@ func (a *App) detectPackageManager() (packageManager, error) { if err != nil { return "", err } - if info.ID == "azurelinux" && info.VariantID == osReleaseIDAzureContainerLinux { + if info.ID == osReleaseIDAzureLinux && info.VariantID == osReleaseIDAzureContainerLinux { return "", fmt.Errorf( "PMC package-based ANC self-update is not supported on image-based OS %q variant %q", info.ID, @@ -282,7 +282,7 @@ func (a *App) detectPackageManager() (packageManager, error) { switch info.ID { case "ubuntu": return pkgMgrApt, nil - case "azurelinux", "mariner": + case osReleaseIDAzureLinux, "mariner": return preferredRpmManager(), nil case osReleaseIDAzureContainerLinux, osReleaseIDFlatcar: // ACL and Flatcar are image-based/immutable distros with no apt/dnf/tdnf. From c8325cc3a89f854145e070cda19bdbdd16ff60d3 Mon Sep 17 00:00:00 2001 From: Devin Wong Date: Wed, 9 Sep 2026 12:37:10 -0700 Subject: [PATCH 16/26] fix: validate renderer dependencies Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- pkg/agent/baker.go | 4 ++- pkg/agent/nodecustomdata_render_test.go | 38 +++++++++++++++++++++---- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/pkg/agent/baker.go b/pkg/agent/baker.go index db56c37de32..944c6a180ad 100644 --- a/pkg/agent/baker.go +++ b/pkg/agent/baker.go @@ -639,7 +639,9 @@ func RenderLinuxNodeCustomDataTemplate(templateContent []byte, config *datamodel config.AgentPoolProfile == nil || config.ContainerService == nil || config.ContainerService.Properties == nil || - config.ContainerService.Properties.OrchestratorProfile == nil { + config.ContainerService.Properties.OrchestratorProfile == nil || + config.K8sComponents == nil || + config.CloudSpecConfig == nil { return "", fmt.Errorf("node bootstrapping configuration is incomplete") } diff --git a/pkg/agent/nodecustomdata_render_test.go b/pkg/agent/nodecustomdata_render_test.go index e271cb4aec7..3417f7f82b7 100644 --- a/pkg/agent/nodecustomdata_render_test.go +++ b/pkg/agent/nodecustomdata_render_test.go @@ -49,13 +49,41 @@ write_files: } } -func TestRenderLinuxNodeCustomDataTemplateRejectsMissingOrchestratorProfile(t *testing.T) { - config := newNodeCustomDataRenderConfig(datamodel.AKSUbuntuContainerd2204Gen2) - config.ContainerService.Properties.OrchestratorProfile = nil +func TestRenderLinuxNodeCustomDataTemplateRejectsMissingDependencies(t *testing.T) { + tests := []struct { + name string + remove func(*datamodel.NodeBootstrappingConfiguration) + }{ + { + name: "orchestrator profile", + remove: func(config *datamodel.NodeBootstrappingConfiguration) { + config.ContainerService.Properties.OrchestratorProfile = nil + }, + }, + { + name: "Kubernetes components", + remove: func(config *datamodel.NodeBootstrappingConfiguration) { + config.K8sComponents = nil + }, + }, + { + name: "cloud spec config", + remove: func(config *datamodel.NodeBootstrappingConfiguration) { + config.CloudSpecConfig = nil + }, + }, + } - _, err := RenderLinuxNodeCustomDataTemplate([]byte("#cloud-config\nwrite_files: []\n"), config) + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + config := newNodeCustomDataRenderConfig(datamodel.AKSUbuntuContainerd2204Gen2) + test.remove(config) - require.EqualError(t, err, "node bootstrapping configuration is incomplete") + _, err := RenderLinuxNodeCustomDataTemplate([]byte("#cloud-config\nwrite_files: []\n"), config) + + require.EqualError(t, err, "node bootstrapping configuration is incomplete") + }) + } } func newNodeCustomDataRenderConfig(distro datamodel.Distro) *datamodel.NodeBootstrappingConfiguration { From f27f33685aecbef1c4fd939a0eca63ad9d7197d5 Mon Sep 17 00:00:00 2001 From: Devin Wong Date: Wed, 9 Sep 2026 15:04:08 -0700 Subject: [PATCH 17/26] refactor(anc): reuse existing applier for embedded script hotfixes Write selected embedded YAML to a temporary file and call the original applyNodeCustomData implementation. Remove transactional replacement machinery and counters while retaining distro gating and fail-open provisioning. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4ae1dd11-ec06-4c26-bfc8-d1ef1f89fcf9 --- aks-node-controller/README.md | 19 +- aks-node-controller/app.go | 29 +- aks-node-controller/app_test.go | 16 +- aks-node-controller/embeddednodecustomdata.go | 74 ++- .../embeddednodecustomdata_test.go | 478 ++++-------------- aks-node-controller/nodecustomdata.go | 342 ++----------- aks-node-controller/nodecustomdata_test.go | 3 - e2e/scenario.go | 2 +- 8 files changed, 183 insertions(+), 780 deletions(-) diff --git a/aks-node-controller/README.md b/aks-node-controller/README.md index 444c8971780..943f0f3c68e 100644 --- a/aks-node-controller/README.md +++ b/aks-node-controller/README.md @@ -153,11 +153,11 @@ Key components: ### Provisioning script hotfix payloads Patched ANC binaries can embed selected Linux provisioning scripts generated from -`parts/linux/cloud-init/artifacts/`. At the start of `provision`, ANC validates the -rendered nodecustomdata matching the local platform and atomically applies its -`write_files` entries before constructing the normal CSE command. -Application is fail-open so the existing VHD scripts remain usable if validation -or replacement fails. +`parts/linux/cloud-init/artifacts/`. At the start of `provision`, ANC writes the +rendered nodecustomdata matching the local platform to a private temporary YAML +file and calls the existing `applyNodeCustomData` function before constructing the +normal CSE command. The temporary YAML is removed afterward. Application errors +are logged and provisioning continues. The embedded nodecustomdata coordinator distinguishes these script hotfixes from updates to the ANC binary itself. The generated files live under @@ -177,10 +177,11 @@ When a PR has no new script hotfix, generation leaves the existing rendered payload unchanged. The active ANC version pointer is likewise retained until it is retired explicitly. -Embedded payloads are replace-only: ANC skips an entry when its runtime -destination does not already exist. File presence preserves non-platform -template gates such as custom-image exclusions. New-file hotfixes are not -supported by this delivery path. +The existing applier writes entries sequentially and creates missing destination +files and parent directories. There is no transactional rollback: if an entry +fails, earlier writes remain. Generation selects by distro only; hotfix authors +must separately account for non-distro template conditions such as custom-image +exclusions. Script hotfix delivery is package-only. The existing base-to-version hotfix map selects the ANC package for the node's baked `YYYYMM.DD` version base; the package diff --git a/aks-node-controller/app.go b/aks-node-controller/app.go index b658e730650..7de307f1fc7 100644 --- a/aks-node-controller/app.go +++ b/aks-node-controller/app.go @@ -72,7 +72,7 @@ type App struct { // is queried. fetchAttestedToken func(ctx context.Context) (string, error) // applyEmbeddedHotfix overrides embedded script application for tests. - applyEmbeddedHotfix func(string) (nodeCustomDataApplyResult, error) + applyEmbeddedHotfix func(string) error // grpcDialContext overrides how the gRPC LPS client dials, letting tests point the client at // an in-process (bufconn) server. When nil, the real TLS dial to the apiserver front is used. grpcDialContext func(ctx context.Context, target string) (net.Conn, error) @@ -686,23 +686,6 @@ func (a *App) Provision(ctx context.Context, flags ProvisionFlags) (*ProvisionRe return provisionResult, err } -func (a *App) applyEmbeddedHotfixIfNeeded() { - applyEmbeddedHotfix := a.applyEmbeddedHotfix - if applyEmbeddedHotfix == nil { - applyEmbeddedHotfix = applyEmbeddedNodeCustomDataIfActive - } - result, err := applyEmbeddedHotfix(a.osReleasePath) - if err != nil { - // Hotfixes are fail-open: the VHD-baked scripts remain available, so an - // embedded payload must not block provisioning. - slog.Warn("failed to apply embedded hotfix payload; continuing with existing scripts", - "error", err) - } else if result.Applied > 0 || result.Skipped > 0 { - slog.Info("processed embedded hotfix payload", - "applied", result.Applied, "skipped", result.Skipped) - } -} - // runProvision encapsulates execution for the "provision" subcommand after CLI parsing. // It returns an error describing any failure; callers should pass that error to // writeCompleteFileOnError so the sentinel file can be written on fail-fast paths. @@ -729,7 +712,15 @@ func (a *App) runProvision(ctx context.Context, flags ProvisionFlags, dryRun boo if dryRun { a.cmdRun = cmdRunnerDryRun } else { - a.applyEmbeddedHotfixIfNeeded() + applyHotfix := a.applyEmbeddedHotfix + if applyHotfix == nil { + applyHotfix = func(osReleasePath string) error { + return applyEmbeddedNodeCustomDataIfActive(embeddedGeneratedNodeCustomData, osReleasePath) + } + } + if err := applyHotfix(a.osReleasePath); err != nil { + slog.Warn("failed to apply embedded hotfix payload; continuing provisioning", "error", err) + } } return a.Provision(ctx, flags) } diff --git a/aks-node-controller/app_test.go b/aks-node-controller/app_test.go index 1e162f92474..4f3a126d0c4 100644 --- a/aks-node-controller/app_test.go +++ b/aks-node-controller/app_test.go @@ -234,9 +234,9 @@ func TestApp_Provision(t *testing.T) { t.Run("embedded hotfix runs before command construction and execution", func(t *testing.T) { tt := NewTestApp(t, TestAppConfig{}) applied := false - tt.App.applyEmbeddedHotfix = func(string) (nodeCustomDataApplyResult, error) { + tt.App.applyEmbeddedHotfix = func(string) error { applied = true - return nodeCustomDataApplyResult{Applied: 1}, nil + return nil } _, err := tt.App.runProvision( @@ -258,8 +258,8 @@ func TestApp_Provision(t *testing.T) { return nil }, }) - tt.App.applyEmbeddedHotfix = func(string) (nodeCustomDataApplyResult, error) { - return nodeCustomDataApplyResult{}, errors.New("rendered nodecustomdata validation failed") + tt.App.applyEmbeddedHotfix = func(string) error { + return errors.New("rendered nodecustomdata application failed") } _, err := tt.App.runProvision( @@ -272,17 +272,17 @@ func TestApp_Provision(t *testing.T) { assert.True(t, executed) assert.Contains(t, logs.getRecords(), logRecord{ Level: slog.LevelWarn, - Message: "failed to apply embedded hotfix payload; continuing with existing scripts", - Attrs: map[string]string{"error": "rendered nodecustomdata validation failed"}, + Message: "failed to apply embedded hotfix payload; continuing provisioning", + Attrs: map[string]string{"error": "rendered nodecustomdata application failed"}, }) }) t.Run("dry-run does not apply embedded hotfix payload", func(t *testing.T) { tt := NewTestApp(t, TestAppConfig{}) applied := false - tt.App.applyEmbeddedHotfix = func(string) (nodeCustomDataApplyResult, error) { + tt.App.applyEmbeddedHotfix = func(string) error { applied = true - return nodeCustomDataApplyResult{}, nil + return nil } _, err := tt.App.runProvision( diff --git a/aks-node-controller/embeddednodecustomdata.go b/aks-node-controller/embeddednodecustomdata.go index 481d5fb79a5..8532a9702a6 100644 --- a/aks-node-controller/embeddednodecustomdata.go +++ b/aks-node-controller/embeddednodecustomdata.go @@ -6,7 +6,6 @@ import ( "io/fs" "log/slog" "os" - "path/filepath" "strings" ) @@ -30,29 +29,52 @@ const ( //go:embed scripthotfix/generated var embeddedGeneratedNodeCustomData embed.FS -//nolint:gochecknoglobals // indirection point so tests can inject an alternate filesystem -var generatedNodeCustomData fs.FS = embeddedGeneratedNodeCustomData - -func applyEmbeddedNodeCustomDataIfActive(osReleasePath string) (nodeCustomDataApplyResult, error) { - active, err := fs.ReadFile(generatedNodeCustomData, "scripthotfix/generated/active") +func applyEmbeddedNodeCustomDataIfActive(payloadFS fs.FS, osReleasePath string) error { + active, err := fs.ReadFile(payloadFS, "scripthotfix/generated/active") if err != nil { - return nodeCustomDataApplyResult{}, fmt.Errorf("read embedded hotfix state: %w", err) + return fmt.Errorf("read embedded hotfix state: %w", err) } if strings.TrimSpace(string(active)) != "true" { - return nodeCustomDataApplyResult{}, nil + return nil } if osReleasePath == "" { osReleasePath = defaultOSReleasePath } platform, err := classifyNodeCustomDataPlatform(osReleasePath) if err != nil { - return nodeCustomDataApplyResult{}, err + return err } if platform == nodeCustomDataPlatformUnsupported { slog.Info("embedded script hotfix is not supported on this OS, skipping", "osReleasePath", osReleasePath) - return nodeCustomDataApplyResult{}, nil + return nil + } + renderedPath := fmt.Sprintf("scripthotfix/generated/rendered_nodecustomdata_%s.yml", platform) + data, err := fs.ReadFile(payloadFS, renderedPath) + if err != nil { + return fmt.Errorf("read embedded nodecustomdata %s: %w", renderedPath, err) + } + temp, err := os.CreateTemp("", "aks-node-controller-nodecustomdata-*.yml") + if err != nil { + return fmt.Errorf("create temporary nodecustomdata: %w", err) + } + defer func() { + if err := os.Remove(temp.Name()); err != nil { + slog.Warn("failed to remove temporary nodecustomdata", "path", temp.Name(), "error", err) + } + }() + _, writeErr := temp.Write(data) + closeErr := temp.Close() + if writeErr != nil { + return fmt.Errorf("write temporary nodecustomdata: %w", writeErr) + } + if closeErr != nil { + return fmt.Errorf("close temporary nodecustomdata: %w", closeErr) } - return applyEmbeddedNodeCustomDataFS(generatedNodeCustomData, platform) + if err := applyNodeCustomData(temp.Name()); err != nil { + return err + } + slog.Info("applied embedded hotfix payload", "source", renderedPath) + return nil } func classifyNodeCustomDataPlatform(osReleasePath string) (nodeCustomDataPlatform, error) { @@ -95,33 +117,3 @@ func parseNodeCustomDataOSRelease(data []byte) map[string]string { } return values } - -func applyEmbeddedNodeCustomDataFS( - payloadFS fs.FS, - platform nodeCustomDataPlatform, -) (nodeCustomDataApplyResult, error) { - if !isConcreteNodeCustomDataPlatform(platform) { - return nodeCustomDataApplyResult{}, fmt.Errorf("unsupported concrete platform %q", platform) - } - renderedPath := filepath.ToSlash(filepath.Join( - "scripthotfix", - "generated", - fmt.Sprintf("rendered_nodecustomdata_%s.yml", platform), - )) - data, err := fs.ReadFile(payloadFS, renderedPath) - if err != nil { - return nodeCustomDataApplyResult{}, fmt.Errorf("read embedded nodecustomdata %s: %w", renderedPath, err) - } - return applyNodeCustomDataPayload(data, nodeCustomDataApplyOptions{ - source: renderedPath, - strict: true, - replaceOnly: true, - requirePermissions: true, - rejectUnsafePaths: true, - rejectEmptyContent: true, - }) -} - -func isConcreteNodeCustomDataPlatform(platform nodeCustomDataPlatform) bool { - return platform == nodeCustomDataPlatformUbuntu || platform == nodeCustomDataPlatformMariner -} diff --git a/aks-node-controller/embeddednodecustomdata_test.go b/aks-node-controller/embeddednodecustomdata_test.go index 537c02fcd87..23fbdf07773 100644 --- a/aks-node-controller/embeddednodecustomdata_test.go +++ b/aks-node-controller/embeddednodecustomdata_test.go @@ -1,13 +1,9 @@ package main import ( - "bytes" - "compress/gzip" "encoding/base64" - "errors" "os" "path/filepath" - "runtime" "testing" "testing/fstest" @@ -22,416 +18,120 @@ func TestClassifyNodeCustomDataPlatform(t *testing.T) { release string expected nodeCustomDataPlatform }{ - {name: "Ubuntu", release: "ID=ubuntu\n", expected: nodeCustomDataPlatformUbuntu}, - {name: "Mariner", release: "ID=mariner\n", expected: nodeCustomDataPlatformMariner}, - {name: "Azure Linux", release: "ID=azurelinux\n", expected: nodeCustomDataPlatformMariner}, - { - name: "OS Guard variant wins over Azure Linux ID", - release: "ID=azurelinux\nVARIANT_ID=osguard\n", - expected: nodeCustomDataPlatformUnsupported, - }, - { - name: "ACL variant wins over Azure Linux ID", - release: "ID=azurelinux\nVARIANT_ID=azurecontainerlinux\n", - expected: nodeCustomDataPlatformUnsupported, - }, - {name: "ACL dedicated ID", release: "ID=azurecontainerlinux\n", expected: nodeCustomDataPlatformUnsupported}, - {name: "Flatcar", release: "ID=flatcar\n", expected: nodeCustomDataPlatformUnsupported}, + {"Ubuntu", "ID=ubuntu\n", nodeCustomDataPlatformUbuntu}, + {"Mariner", "ID=mariner\n", nodeCustomDataPlatformMariner}, + {"Azure Linux", "ID=azurelinux\n", nodeCustomDataPlatformMariner}, + {"OS Guard", "ID=azurelinux\nVARIANT_ID=osguard\n", nodeCustomDataPlatformUnsupported}, + {"ACL variant", "ID=azurelinux\nVARIANT_ID=azurecontainerlinux\n", nodeCustomDataPlatformUnsupported}, + {"ACL ID", "ID=azurecontainerlinux\n", nodeCustomDataPlatformUnsupported}, + {"Flatcar", "ID=flatcar\n", nodeCustomDataPlatformUnsupported}, + {"Quoted OS Guard", "ID=\"AZURELINUX\"\nVARIANT_ID=\"OSGUARD\"\n", nodeCustomDataPlatformUnsupported}, } - for _, test := range tests { t.Run(test.name, func(t *testing.T) { releasePath := filepath.Join(t.TempDir(), "os-release") require.NoError(t, os.WriteFile(releasePath, []byte(test.release), 0o600)) - actual, err := classifyNodeCustomDataPlatform(releasePath) - require.NoError(t, err) assert.Equal(t, test.expected, actual) - }) - } - - t.Run("unsupported ID fails explicitly", func(t *testing.T) { - releasePath := filepath.Join(t.TempDir(), "os-release") - require.NoError(t, os.WriteFile(releasePath, []byte("ID=other\n"), 0o600)) - - _, err := classifyNodeCustomDataPlatform(releasePath) - - require.ErrorContains(t, err, "unsupported OS ID") - }) -} - -func TestApplyEmbeddedNodeCustomDataIfActiveSkipsInactivePayload(t *testing.T) { - original := generatedNodeCustomData - generatedNodeCustomData = fstest.MapFS{ - "scripthotfix/generated/active": &fstest.MapFile{Data: []byte("false\n")}, - } - t.Cleanup(func() { - generatedNodeCustomData = original - }) - - result, err := applyEmbeddedNodeCustomDataIfActive(filepath.Join(t.TempDir(), "missing-os-release")) - - require.NoError(t, err) - assert.Equal(t, nodeCustomDataApplyResult{}, result) -} - -func TestApplyEmbeddedNodeCustomDataIfActiveSkipsUnsupportedPlatforms(t *testing.T) { - original := generatedNodeCustomData - generatedNodeCustomData = fstest.MapFS{ - "scripthotfix/generated/active": &fstest.MapFile{Data: []byte("true\n")}, - // A variant accidentally classified as Mariner must fail, not silently pass. - embeddedRenderedPath(nodeCustomDataPlatformMariner): &fstest.MapFile{Data: []byte("invalid: [")}, - } - t.Cleanup(func() { - generatedNodeCustomData = original - }) - - for _, release := range []string{ - "ID=azurelinux\nVARIANT_ID=osguard\n", - "ID=azurelinux\nVARIANT_ID=azurecontainerlinux\n", - "ID=azurecontainerlinux\n", - "ID=flatcar\n", - "ID=\"AZURELINUX\"\nVARIANT_ID=\"OSGUARD\"\n", - } { - t.Run(release, func(t *testing.T) { - releasePath := filepath.Join(t.TempDir(), "os-release") - require.NoError(t, os.WriteFile(releasePath, []byte(release), 0o600)) - - result, err := applyEmbeddedNodeCustomDataIfActive(releasePath) - - require.NoError(t, err) - assert.Equal(t, nodeCustomDataApplyResult{}, result) - }) - } -} - -func TestApplyEmbeddedNodeCustomDataFSRejectsUnsupportedPlatforms(t *testing.T) { - for _, platform := range []nodeCustomDataPlatform{"acl", "azlosguard", "flatcar", nodeCustomDataPlatformUnsupported} { - t.Run(string(platform), func(t *testing.T) { - _, err := applyEmbeddedNodeCustomDataFS(fstest.MapFS{}, platform) - require.ErrorContains(t, err, "unsupported concrete platform") + if actual == nodeCustomDataPlatformUnsupported { + files := fstest.MapFS{ + "scripthotfix/generated/active": &fstest.MapFile{Data: []byte("true\n")}, + } + require.NoError(t, applyEmbeddedNodeCustomDataIfActive(files, releasePath), + "unsupported platforms must skip without reading any payload") + } }) } } -func TestApplyEmbeddedNodeCustomDataIfActiveSelectsPlatformPayload(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("Windows rename cannot atomically replace an existing destination") - } - - for _, test := range []struct { - id string - platform nodeCustomDataPlatform - }{ - {id: "ubuntu", platform: nodeCustomDataPlatformUbuntu}, - {id: "mariner", platform: nodeCustomDataPlatformMariner}, - {id: "azurelinux", platform: nodeCustomDataPlatformMariner}, - } { - t.Run(test.id, func(t *testing.T) { +func TestApplyEmbeddedNodeCustomData(t *testing.T) { + for _, id := range []string{"ubuntu", "mariner", "azurelinux"} { + t.Run(id, func(t *testing.T) { directory := t.TempDir() - destination := filepath.Join(directory, "provision.sh") - require.NoError(t, os.WriteFile(destination, []byte("old"), 0o600)) + t.Setenv("TMPDIR", directory) + t.Setenv("TMP", directory) releasePath := filepath.Join(directory, "os-release") - require.NoError(t, os.WriteFile(releasePath, []byte("ID="+test.id+"\n"), 0o600)) - payload := []byte("#!/bin/sh\necho fixed\n") - - original := generatedNodeCustomData - generatedNodeCustomData = fstest.MapFS{ - "scripthotfix/generated/active": &fstest.MapFile{Data: []byte("true\n")}, - embeddedRenderedPath(test.platform): &fstest.MapFile{Data: marshalNodeCustomData(t, []nodeCustomDataWriteFile{{ - Path: destination, - Permissions: "0744", - Encoding: encodingBase64, - Owner: "root", - Content: base64.StdEncoding.EncodeToString(payload), - }})}, - } - t.Cleanup(func() { - generatedNodeCustomData = original - }) - - result, err := applyEmbeddedNodeCustomDataIfActive(releasePath) - + require.NoError(t, os.WriteFile(releasePath, []byte("ID="+id+"\n"), 0o600)) + existing := filepath.Join(directory, "existing.sh") + missing := filepath.Join(directory, "new", "script.sh") + require.NoError(t, os.WriteFile(existing, []byte("old"), 0o600)) + payload := "echo hotfixed\n" + data, err := yaml.Marshal(nodeCustomData{WriteFiles: []nodeCustomDataWriteFile{ + {Path: existing, Encoding: encodingBase64, Content: base64.StdEncoding.EncodeToString([]byte(payload))}, + {Path: missing, Permissions: "0744", Content: payload}, + }}) require.NoError(t, err) - assert.Equal(t, nodeCustomDataApplyResult{Applied: 1}, result) - actual, err := os.ReadFile(destination) + platform := id + if id == osReleaseIDAzureLinux { + platform = string(nodeCustomDataPlatformMariner) + } + files := fstest.MapFS{ + "scripthotfix/generated/active": &fstest.MapFile{Data: []byte("true\n")}, + "scripthotfix/generated/rendered_nodecustomdata_" + platform + ".yml": &fstest.MapFile{Data: data}, + } + require.NoError(t, applyEmbeddedNodeCustomDataIfActive(files, releasePath)) + for _, destination := range []string{existing, missing} { + actual, readErr := os.ReadFile(destination) + require.NoError(t, readErr) + assert.Equal(t, payload, string(actual)) + } + temporary, err := filepath.Glob(filepath.Join(directory, "aks-node-controller-nodecustomdata-*.yml")) require.NoError(t, err) - assert.Equal(t, payload, actual) + assert.Empty(t, temporary) }) } } -func TestApplyEmbeddedNodeCustomDataIsReplaceOnlyAndIdempotent(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("Windows rename cannot atomically replace an existing destination") - } - - directory := t.TempDir() - destination := filepath.Join(directory, "provision.sh") - missing := filepath.Join(directory, "missing.sh") - require.NoError(t, os.WriteFile(destination, []byte("old"), 0o600)) - files := embeddedRenderedFS(t, nodeCustomDataPlatformUbuntu, []nodeCustomDataWriteFile{ - { - Path: destination, - Permissions: "0744", - Owner: "root", - Content: "hotfix", - }, - { - Path: missing, - Permissions: "0744", - Owner: "root", - Content: "not-created", - }, - }) - - first, err := applyEmbeddedNodeCustomDataFS(files, nodeCustomDataPlatformUbuntu) - - require.NoError(t, err) - assert.Equal(t, nodeCustomDataApplyResult{Applied: 1, Skipped: 1}, first) - actual, err := os.ReadFile(destination) - require.NoError(t, err) - assert.Equal(t, []byte("hotfix"), actual) - info, err := os.Stat(destination) - require.NoError(t, err) - assert.Equal(t, os.FileMode(0o744), info.Mode().Perm()) - _, statErr := os.Stat(missing) - assert.True(t, os.IsNotExist(statErr)) - - second, err := applyEmbeddedNodeCustomDataFS(files, nodeCustomDataPlatformUbuntu) - require.NoError(t, err) - assert.Equal(t, nodeCustomDataApplyResult{Skipped: 2}, second) -} - -func TestEmbeddedNodeCustomDataStrictValidation(t *testing.T) { - validDestination := filepath.Join(t.TempDir(), "provision.sh") - valid := nodeCustomDataWriteFile{ - Path: validDestination, - Permissions: "0744", - Owner: "root", - Content: "hotfix", - } - +func TestApplyEmbeddedNodeCustomDataErrorsAndCleanup(t *testing.T) { tests := []struct { - name string - files []nodeCustomDataWriteFile - expectedErr string + name string + active string + release string + payload string + missing string + wantError string }{ - { - name: "unsafe destination", - files: []nodeCustomDataWriteFile{{ - Path: "../provision.sh", - Permissions: "0744", - Owner: "root", - Content: "hotfix", - }}, - expectedErr: "unsafe destination", - }, - { - name: "destination with embedded backslash", - files: []nodeCustomDataWriteFile{{ - Path: `/opt/provision\script.sh`, - Permissions: "0744", - Owner: "root", - Content: "hotfix", - }}, - expectedErr: "backslashes are not allowed", - }, - { - name: "absent mode", - files: []nodeCustomDataWriteFile{{ - Path: validDestination, - Owner: "root", - Content: "hotfix", - }}, - expectedErr: "invalid mode", - }, - { - name: "invalid mode", - files: []nodeCustomDataWriteFile{{ - Path: validDestination, - Permissions: "0999", - Owner: "root", - Content: "hotfix", - }}, - expectedErr: "invalid mode", - }, - { - name: "unsupported owner", - files: []nodeCustomDataWriteFile{{ - Path: validDestination, - Permissions: "0744", - Owner: "nobody", - Content: "hotfix", - }}, - expectedErr: "unsupported owner", - }, - { - name: "unsupported encoding", - files: []nodeCustomDataWriteFile{{ - Path: validDestination, - Permissions: "0744", - Owner: "root", - Encoding: "rot13", - Content: "hotfix", - }}, - expectedErr: "unsupported encoding", - }, - { - name: "empty decoded content", - files: []nodeCustomDataWriteFile{{ - Path: validDestination, - Permissions: "0744", - Owner: "root", - Encoding: encodingBase64, - Content: "", - }}, - expectedErr: "is empty", - }, - { - name: "duplicate destination", - files: []nodeCustomDataWriteFile{valid, valid}, - expectedErr: "duplicate destination", - }, + {name: "inactive skips missing OS release", active: "false\n", missing: "release"}, + {name: "missing active", missing: "active", wantError: "read embedded hotfix state"}, + {name: "missing release", active: "true", missing: "release", wantError: "read OS release"}, + {name: "unknown OS", active: "true", release: "ID=other", wantError: "unsupported OS ID"}, + {name: "missing ID", active: "true", release: "VERSION_ID=3.0", wantError: "ID is missing"}, + {name: "missing payload", active: "true", release: "ID=ubuntu", missing: "payload", wantError: "read embedded nodecustomdata"}, + {name: "malformed YAML", active: "true", release: "ID=ubuntu", payload: "write_files: [", wantError: "unmarshal nodecustomdata"}, + {name: "invalid entry", active: "true", release: "ID=ubuntu", payload: "write_files:\n- content: invalid\n", wantError: "path is required"}, + {name: "empty payload", active: "true", release: "ID=ubuntu", payload: "write_files: []\n"}, + {name: "temporary directory unavailable", active: "true", release: "ID=ubuntu", payload: "write_files: []\n", missing: "temp", wantError: "create temporary nodecustomdata"}, } - for _, test := range tests { t.Run(test.name, func(t *testing.T) { - _, err := applyEmbeddedNodeCustomDataFS( - embeddedRenderedFS(t, nodeCustomDataPlatformUbuntu, test.files), - nodeCustomDataPlatformUbuntu, - ) - require.ErrorContains(t, err, test.expectedErr) - }) - } - - t.Run("unknown YAML field", func(t *testing.T) { - files := fstest.MapFS{ - embeddedRenderedPath(nodeCustomDataPlatformUbuntu): &fstest.MapFile{ - Data: []byte("write_files: []\nunknown: true\n"), - }, - } - _, err := applyEmbeddedNodeCustomDataFS(files, nodeCustomDataPlatformUbuntu) - require.ErrorContains(t, err, "field unknown not found") - }) - - t.Run("trailing YAML document", func(t *testing.T) { - files := fstest.MapFS{ - embeddedRenderedPath(nodeCustomDataPlatformUbuntu): &fstest.MapFile{ - Data: []byte("write_files: []\n---\nwrite_files: []\n"), - }, - } - _, err := applyEmbeddedNodeCustomDataFS(files, nodeCustomDataPlatformUbuntu) - require.ErrorContains(t, err, "trailing content") - }) -} - -func TestNodeCustomDataSharedDecoderHandlesGzip(t *testing.T) { - var compressed bytes.Buffer - writer := gzip.NewWriter(&compressed) - _, err := writer.Write([]byte("rendered payload")) - require.NoError(t, err) - require.NoError(t, writer.Close()) - - decoded, err := decodeNodeCustomDataWriteFileContent(nodeCustomDataWriteFile{ - Encoding: encodingGZIP, - Content: compressed.String(), - }) - - require.NoError(t, err) - assert.Equal(t, []byte("rendered payload"), decoded) -} - -func TestEmbeddedNodeCustomDataStagesAllBeforeCommit(t *testing.T) { - directory := t.TempDir() - firstDestination := filepath.Join(directory, "first.sh") - require.NoError(t, os.WriteFile(firstDestination, []byte("original"), 0o700)) - files := embeddedRenderedFS(t, nodeCustomDataPlatformUbuntu, []nodeCustomDataWriteFile{ - { - Path: firstDestination, - Permissions: "0744", - Owner: "root", - Content: "first hotfix", - }, - { - Path: directory, - Permissions: "0744", - Owner: "root", - Content: "second hotfix", - }, - }) - - _, err := applyEmbeddedNodeCustomDataFS(files, nodeCustomDataPlatformUbuntu) - - require.ErrorContains(t, err, "read destination") - actual, readErr := os.ReadFile(firstDestination) - require.NoError(t, readErr) - assert.Equal(t, []byte("original"), actual) - staged, globErr := filepath.Glob(filepath.Join(directory, ".aks-node-controller-nodecustomdata-*")) - require.NoError(t, globErr) - assert.Empty(t, staged) -} - -func TestCommitStagedNodeCustomDataRollsBackEarlierReplacement(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("Windows rename cannot atomically replace an existing destination") - } - directory := t.TempDir() - firstDestination := filepath.Join(directory, "first.sh") - secondDestination := filepath.Join(directory, "second.sh") - require.NoError(t, os.WriteFile(firstDestination, []byte("first original"), 0o700)) - require.NoError(t, os.WriteFile(secondDestination, []byte("second original"), 0o711)) - first, changed, err := stageNodeCustomDataEntry( - nodeCustomDataEntry{destination: firstDestination, content: []byte("first hotfix"), mode: 0o744}, - true, - ) - require.NoError(t, err) - require.True(t, changed) - second, changed, err := stageNodeCustomDataEntry( - nodeCustomDataEntry{destination: secondDestination, content: []byte("second hotfix"), mode: 0o755}, - true, - ) - require.NoError(t, err) - require.True(t, changed) - - err = commitStagedNodeCustomDataWithRename( - []*stagedNodeCustomDataEntry{&first, &second}, - func(source string, destination string) error { - if source == second.stagedPath { - return errors.New("injected rename failure") + directory := t.TempDir() + tempDir := directory + if test.missing == "temp" { + tempDir = filepath.Join(directory, "missing") } - return os.Rename(source, destination) - }, - ) - - require.ErrorContains(t, err, "injected rename failure") - firstActual, readErr := os.ReadFile(firstDestination) - require.NoError(t, readErr) - assert.Equal(t, []byte("first original"), firstActual) - secondActual, readErr := os.ReadFile(secondDestination) - require.NoError(t, readErr) - assert.Equal(t, []byte("second original"), secondActual) -} - -func embeddedRenderedFS( - t *testing.T, - platform nodeCustomDataPlatform, - files []nodeCustomDataWriteFile, -) fstest.MapFS { - t.Helper() - return fstest.MapFS{ - embeddedRenderedPath(platform): &fstest.MapFile{Data: marshalNodeCustomData(t, files)}, + t.Setenv("TMPDIR", tempDir) + t.Setenv("TMP", tempDir) + releasePath := filepath.Join(directory, "os-release") + if test.missing != "release" { + require.NoError(t, os.WriteFile(releasePath, []byte(test.release), 0o600)) + } + files := fstest.MapFS{} + if test.missing != "active" { + files["scripthotfix/generated/active"] = &fstest.MapFile{Data: []byte(test.active)} + } + if test.missing != "payload" { + files["scripthotfix/generated/rendered_nodecustomdata_ubuntu.yml"] = &fstest.MapFile{Data: []byte(test.payload)} + } + err := applyEmbeddedNodeCustomDataIfActive(files, releasePath) + if test.wantError == "" { + require.NoError(t, err) + } else { + require.ErrorContains(t, err, test.wantError) + } + temporary, err := filepath.Glob(filepath.Join(directory, "aks-node-controller-nodecustomdata-*.yml")) + require.NoError(t, err) + assert.Empty(t, temporary) + }) } } - -func marshalNodeCustomData(t *testing.T, files []nodeCustomDataWriteFile) []byte { - t.Helper() - data, err := yaml.Marshal(nodeCustomData{WriteFiles: files}) - require.NoError(t, err) - return data -} - -func embeddedRenderedPath(platform nodeCustomDataPlatform) string { - return "scripthotfix/generated/rendered_nodecustomdata_" + string(platform) + ".yml" -} diff --git a/aks-node-controller/nodecustomdata.go b/aks-node-controller/nodecustomdata.go index bc72ba5926e..0d178d81e59 100644 --- a/aks-node-controller/nodecustomdata.go +++ b/aks-node-controller/nodecustomdata.go @@ -4,14 +4,11 @@ import ( "bytes" "compress/gzip" "encoding/base64" - "errors" "fmt" "io" "os" - "path" "path/filepath" "strconv" - "strings" "gopkg.in/yaml.v3" ) @@ -34,169 +31,60 @@ type nodeCustomData struct { WriteFiles []nodeCustomDataWriteFile `yaml:"write_files"` } -type nodeCustomDataApplyOptions struct { - source string - strict bool - replaceOnly bool - requirePermissions bool - rejectUnsafePaths bool - rejectEmptyContent bool -} - -type nodeCustomDataApplyResult struct { - Applied int - Skipped int -} - -type nodeCustomDataEntry struct { - destination string - mode os.FileMode - content []byte -} - -type stagedNodeCustomDataEntry struct { - destination string - stagedPath string - backupPath string - originalExist bool - preserveBackup bool -} - -func applyNodeCustomData(nodeCustomDataPath string) error { - data, err := os.ReadFile(nodeCustomDataPath) +func applyNodeCustomData(path string) error { + data, err := os.ReadFile(path) if err != nil { if os.IsNotExist(err) { return nil } - return fmt.Errorf("read nodecustomdata %s: %w", nodeCustomDataPath, err) - } - - if _, err := applyNodeCustomDataPayload(data, nodeCustomDataApplyOptions{source: nodeCustomDataPath}); err != nil { - return fmt.Errorf("apply nodecustomdata %s: %w", nodeCustomDataPath, err) - } - return nil -} - -func applyNodeCustomDataPayload(data []byte, options nodeCustomDataApplyOptions) (nodeCustomDataApplyResult, error) { - customData, err := parseNodeCustomData(data, options) - if err != nil { - return nodeCustomDataApplyResult{}, err + return fmt.Errorf("read nodecustomdata %s: %w", path, err) } - entries, err := validateNodeCustomData(customData, options) - if err != nil { - return nodeCustomDataApplyResult{}, err + var customData nodeCustomData + if err := yaml.Unmarshal(data, &customData); err != nil { + return fmt.Errorf("unmarshal nodecustomdata %s: %w", path, err) } - result := nodeCustomDataApplyResult{} - var staged []*stagedNodeCustomDataEntry - for _, entry := range entries { - pending, changed, err := stageNodeCustomDataEntry(entry, options.replaceOnly) - if err != nil { - cleanupStagedNodeCustomData(staged) - return result, fmt.Errorf("stage nodecustomdata destination %s: %w", entry.destination, err) + for _, file := range customData.WriteFiles { + if err := applyNodeCustomDataWriteFile(file); err != nil { + return fmt.Errorf("apply nodecustomdata write file %s: %w", file.Path, err) } - if !changed { - result.Skipped++ - continue - } - staged = append(staged, &pending) } - if err := commitStagedNodeCustomData(staged); err != nil { - return nodeCustomDataApplyResult{}, err - } - result.Applied = len(staged) - return result, nil + return nil } -func parseNodeCustomData(data []byte, options nodeCustomDataApplyOptions) (nodeCustomData, error) { - var customData nodeCustomData - if !options.strict { - if err := yaml.Unmarshal(data, &customData); err != nil { - return nodeCustomData{}, fmt.Errorf("unmarshal nodecustomdata %s: %w", options.source, err) - } - return customData, nil - } - - decoder := yaml.NewDecoder(bytes.NewReader(data)) - decoder.KnownFields(true) - if err := decoder.Decode(&customData); err != nil { - return nodeCustomData{}, fmt.Errorf("decode nodecustomdata %s: %w", options.source, err) +func applyNodeCustomDataWriteFile(file nodeCustomDataWriteFile) error { + if file.Path == "" { + return fmt.Errorf("path is required") } - var trailing any - if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { - return nodeCustomData{}, fmt.Errorf("nodecustomdata %s has trailing content", options.source) + if file.Owner != "" && file.Owner != "root" { + return fmt.Errorf("unsupported owner %q", file.Owner) } - return customData, nil -} -func validateNodeCustomData(customData nodeCustomData, options nodeCustomDataApplyOptions) ([]nodeCustomDataEntry, error) { - entries := make([]nodeCustomDataEntry, 0, len(customData.WriteFiles)) - destinations := make(map[string]struct{}, len(customData.WriteFiles)) - for index, file := range customData.WriteFiles { - entry, err := validateNodeCustomDataWriteFile(file, options) + mode := os.FileMode(0o644) + if file.Permissions != "" { + parsedMode, err := strconv.ParseUint(file.Permissions, 8, 32) if err != nil { - return nil, fmt.Errorf("validate write_files entry %d: %w", index, err) - } - if options.strict { - if _, exists := destinations[entry.destination]; exists { - return nil, fmt.Errorf("duplicate destination %s", entry.destination) - } - destinations[entry.destination] = struct{}{} + return fmt.Errorf("parse permissions: %w", err) } - entries = append(entries, entry) + mode = os.FileMode(parsedMode) } - return entries, nil -} -func validateNodeCustomDataWriteFile(file nodeCustomDataWriteFile, options nodeCustomDataApplyOptions) (nodeCustomDataEntry, error) { - if file.Path == "" { - return nodeCustomDataEntry{}, fmt.Errorf("path is required") - } - if options.rejectUnsafePaths { - if (!strings.HasPrefix(file.Path, "/") && !filepath.IsAbs(file.Path)) || - (strings.HasPrefix(file.Path, "/") && path.Clean(file.Path) != file.Path) || - (!strings.HasPrefix(file.Path, "/") && filepath.Clean(file.Path) != file.Path) { - return nodeCustomDataEntry{}, fmt.Errorf("unsafe destination %q", file.Path) - } - if strings.HasPrefix(file.Path, "/") && strings.Contains(file.Path, `\`) { - return nodeCustomDataEntry{}, fmt.Errorf("unsafe destination %q: backslashes are not allowed", file.Path) - } - } - if file.Owner != "" && file.Owner != "root" { - return nodeCustomDataEntry{}, fmt.Errorf("unsupported owner %q", file.Owner) - } - - mode, err := parseNodeCustomDataMode(file.Permissions, options.requirePermissions) - if err != nil { - return nodeCustomDataEntry{}, err - } - content, err := decodeNodeCustomDataWriteFileContent(file) + contents, err := decodeNodeCustomDataWriteFileContent(file) if err != nil { - return nodeCustomDataEntry{}, err - } - if options.rejectEmptyContent && len(content) == 0 { - return nodeCustomDataEntry{}, fmt.Errorf("content for %s is empty", file.Path) + return err } - return nodeCustomDataEntry{destination: file.Path, mode: mode, content: content}, nil -} -func parseNodeCustomDataMode(value string, required bool) (os.FileMode, error) { - if value == "" && !required { - return 0o644, nil + if err := os.MkdirAll(filepath.Dir(file.Path), 0o755); err != nil { + return fmt.Errorf("create parent directory: %w", err) } - parsed, err := strconv.ParseUint(value, 8, 32) - if err != nil { - if required { - return 0, fmt.Errorf("invalid mode %q", value) - } - return 0, fmt.Errorf("parse permissions: %w", err) - } - if required && (parsed == 0 || parsed > 0o777) { - return 0, fmt.Errorf("invalid mode %q", value) + + if err := os.WriteFile(file.Path, contents, mode); err != nil { + return fmt.Errorf("write file: %w", err) } - return os.FileMode(parsed), nil + + return nil } func decodeNodeCustomDataWriteFileContent(file nodeCustomDataWriteFile) ([]byte, error) { @@ -206,188 +94,22 @@ func decodeNodeCustomDataWriteFileContent(file nodeCustomDataWriteFile) ([]byte, case encodingGZIP: reader, err := gzip.NewReader(bytes.NewReader([]byte(file.Content))) if err != nil { - return nil, fmt.Errorf("create gzip reader for %s: %w", file.Path, err) + return nil, fmt.Errorf("create gzip reader: %w", err) } defer reader.Close() decoded, err := io.ReadAll(reader) if err != nil { - return nil, fmt.Errorf("read gzip content for %s: %w", file.Path, err) + return nil, fmt.Errorf("read gzip content: %w", err) } return decoded, nil case encodingBase64: decoded, err := base64.StdEncoding.DecodeString(file.Content) if err != nil { - return nil, fmt.Errorf("decode base64 content for %s: %w", file.Path, err) + return nil, fmt.Errorf("decode base64 content: %w", err) } return decoded, nil default: return nil, fmt.Errorf("unsupported encoding %q", file.Encoding) } } - -func stageNodeCustomDataEntry(entry nodeCustomDataEntry, replaceOnly bool) (stagedNodeCustomDataEntry, bool, error) { - current, err := os.ReadFile(entry.destination) - originalExists := err == nil - var originalMode os.FileMode - switch { - case err == nil: - info, statErr := os.Stat(entry.destination) - if statErr != nil { - return stagedNodeCustomDataEntry{}, false, fmt.Errorf("stat destination: %w", statErr) - } - originalMode = info.Mode().Perm() - if bytes.Equal(current, entry.content) && originalMode == entry.mode.Perm() { - return stagedNodeCustomDataEntry{}, false, nil - } - case os.IsNotExist(err): - if replaceOnly { - return stagedNodeCustomDataEntry{}, false, nil - } - default: - return stagedNodeCustomDataEntry{}, false, fmt.Errorf("read destination: %w", err) - } - - directory := filepath.Dir(entry.destination) - if originalExists { - info, statErr := os.Stat(directory) - if statErr != nil { - return stagedNodeCustomDataEntry{}, false, fmt.Errorf("stat destination directory %s: %w", directory, statErr) - } - if !info.IsDir() { - return stagedNodeCustomDataEntry{}, false, fmt.Errorf("destination parent %s is not a directory", directory) - } - } else if mkErr := os.MkdirAll(directory, 0o755); mkErr != nil { - return stagedNodeCustomDataEntry{}, false, fmt.Errorf("create parent directory: %w", mkErr) - } - - stagedPath, err := writeNodeCustomDataTempFile(directory, ".aks-node-controller-nodecustomdata-stage-*", entry.content, entry.mode) - if err != nil { - return stagedNodeCustomDataEntry{}, false, err - } - staged := stagedNodeCustomDataEntry{ - destination: entry.destination, - stagedPath: stagedPath, - originalExist: originalExists, - } - if !originalExists { - return staged, true, nil - } - - backupPath, err := writeNodeCustomDataTempFile( - directory, - ".aks-node-controller-nodecustomdata-backup-*", - current, - originalMode, - ) - if err != nil { - _ = os.Remove(stagedPath) - return stagedNodeCustomDataEntry{}, false, fmt.Errorf("stage destination backup: %w", err) - } - staged.backupPath = backupPath - return staged, true, nil -} - -func writeNodeCustomDataTempFile(directory, pattern string, content []byte, mode os.FileMode) (string, error) { - temp, err := os.CreateTemp(directory, pattern) - if err != nil { - return "", fmt.Errorf("create temporary file: %w", err) - } - tempPath := temp.Name() - cleanup := func() { - _ = temp.Close() - _ = os.Remove(tempPath) - } - if _, err := temp.Write(content); err != nil { - cleanup() - return "", fmt.Errorf("write temporary file: %w", err) - } - // Chmod before Sync so the final mode is included in the fsync; otherwise a - // crash after the later rename could durably leave the script with - // CreateTemp's 0600 (non-executable) mode. - if err := temp.Chmod(mode); err != nil { - cleanup() - return "", fmt.Errorf("chmod temporary file: %w", err) - } - if err := temp.Sync(); err != nil { - cleanup() - return "", fmt.Errorf("sync temporary file: %w", err) - } - if err := temp.Close(); err != nil { - cleanup() - return "", fmt.Errorf("close temporary file: %w", err) - } - return tempPath, nil -} - -func commitStagedNodeCustomData(staged []*stagedNodeCustomDataEntry) error { - return commitStagedNodeCustomDataWithRename(staged, os.Rename) -} - -func commitStagedNodeCustomDataWithRename( - staged []*stagedNodeCustomDataEntry, - rename func(string, string) error, -) error { - committed := 0 - defer cleanupStagedNodeCustomData(staged) - for index, entry := range staged { - if err := rename(entry.stagedPath, entry.destination); err != nil { - rollbackErr := rollbackStagedNodeCustomData(staged[:committed], rename) - if rollbackErr != nil { - return fmt.Errorf( - "commit nodecustomdata destination %s: %w; rollback failed: %w", - entry.destination, - err, - rollbackErr, - ) - } - return fmt.Errorf("commit nodecustomdata destination %s: %w", entry.destination, err) - } - staged[index].stagedPath = "" - committed++ - } - return nil -} - -func rollbackStagedNodeCustomData( - committed []*stagedNodeCustomDataEntry, - rename func(string, string) error, -) error { - var rollbackErrors []error - for index := len(committed) - 1; index >= 0; index-- { - entry := committed[index] - var err error - if entry.originalExist { - err = rename(entry.backupPath, entry.destination) - if err == nil { - entry.backupPath = "" - } - } else { - err = os.Remove(entry.destination) - } - if err != nil { - entry.preserveBackup = true - rollbackErrors = append( - rollbackErrors, - fmt.Errorf( - "restore %s from preserved backup %s: %w", - entry.destination, - entry.backupPath, - err, - ), - ) - } - } - return errors.Join(rollbackErrors...) -} - -func cleanupStagedNodeCustomData(staged []*stagedNodeCustomDataEntry) { - for _, entry := range staged { - if entry.stagedPath != "" { - _ = os.Remove(entry.stagedPath) - } - if entry.backupPath != "" && !entry.preserveBackup { - _ = os.Remove(entry.backupPath) - } - } -} diff --git a/aks-node-controller/nodecustomdata_test.go b/aks-node-controller/nodecustomdata_test.go index 65e6192a87b..bd050423cfa 100644 --- a/aks-node-controller/nodecustomdata_test.go +++ b/aks-node-controller/nodecustomdata_test.go @@ -72,9 +72,6 @@ func TestApplyNodeCustomDataPreservesLegacyDefaultsAndCreatesParents(t *testing. actual, err := os.ReadFile(destination) require.NoError(t, err) assert.Equal(t, []byte("base64-content"), actual) - mode, err := parseNodeCustomDataMode("", false) - require.NoError(t, err) - assert.Equal(t, os.FileMode(0o644), mode) if runtime.GOOS != "windows" { info, statErr := os.Stat(destination) require.NoError(t, statErr) diff --git a/e2e/scenario.go b/e2e/scenario.go index 1fe8ed0d918..51708b62775 100644 --- a/e2e/scenario.go +++ b/e2e/scenario.go @@ -740,7 +740,7 @@ func newUbuntu2204EmbeddedScriptHotfixScenario() *Scenario { ctx, s, "/var/log/azure/aks-node-controller.output", - "processed embedded hotfix payload", + "applied embedded hotfix payload", ), ) }, From 18a6667117ff2d07d16841622262e828b9a3820d Mon Sep 17 00:00:00 2001 From: Devin Wong Date: Wed, 9 Sep 2026 15:14:39 -0700 Subject: [PATCH 18/26] refactor(anc): trim redundant hotfix code and tests Remove unreachable empty-template output, duplicate source checks, obsolete fixture platforms, and diagnostic-only OS handling. Restore legacy applier tests to baseline and retain default-mode coverage in the embedded test. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4ae1dd11-ec06-4c26-bfc8-d1ef1f89fcf9 --- .../embeddednodecustomdata_test.go | 8 ++++- aks-node-controller/hotfix.go | 5 ---- aks-node-controller/hotfix_test.go | 12 -------- aks-node-controller/nodecustomdata_test.go | 30 ------------------- e2e/vmss.go | 7 ++--- hotfix/hotfix_generate.py | 9 +----- hotfix/hotfix_generate_test.py | 8 ----- 7 files changed, 10 insertions(+), 69 deletions(-) diff --git a/aks-node-controller/embeddednodecustomdata_test.go b/aks-node-controller/embeddednodecustomdata_test.go index 23fbdf07773..0ea4edcf8d0 100644 --- a/aks-node-controller/embeddednodecustomdata_test.go +++ b/aks-node-controller/embeddednodecustomdata_test.go @@ -4,6 +4,7 @@ import ( "encoding/base64" "os" "path/filepath" + "runtime" "testing" "testing/fstest" @@ -59,7 +60,7 @@ func TestApplyEmbeddedNodeCustomData(t *testing.T) { payload := "echo hotfixed\n" data, err := yaml.Marshal(nodeCustomData{WriteFiles: []nodeCustomDataWriteFile{ {Path: existing, Encoding: encodingBase64, Content: base64.StdEncoding.EncodeToString([]byte(payload))}, - {Path: missing, Permissions: "0744", Content: payload}, + {Path: missing, Content: payload}, }}) require.NoError(t, err) platform := id @@ -79,6 +80,11 @@ func TestApplyEmbeddedNodeCustomData(t *testing.T) { temporary, err := filepath.Glob(filepath.Join(directory, "aks-node-controller-nodecustomdata-*.yml")) require.NoError(t, err) assert.Empty(t, temporary) + if runtime.GOOS != "windows" { + info, statErr := os.Stat(missing) + require.NoError(t, statErr) + assert.Equal(t, os.FileMode(0o644), info.Mode().Perm()) + } }) } } diff --git a/aks-node-controller/hotfix.go b/aks-node-controller/hotfix.go index 353dacb0efd..6291c854ad0 100644 --- a/aks-node-controller/hotfix.go +++ b/aks-node-controller/hotfix.go @@ -284,11 +284,6 @@ func (a *App) detectPackageManager() (packageManager, error) { return pkgMgrApt, nil case osReleaseIDAzureLinux, "mariner": return preferredRpmManager(), nil - case osReleaseIDAzureContainerLinux, osReleaseIDFlatcar: - // ACL and Flatcar are image-based/immutable distros with no apt/dnf/tdnf. - // ANC self-update via a PMC package is intentionally unsupported there; - // they only ever run the ANC binary baked into the VHD. - return "", fmt.Errorf("PMC package-based ANC self-update is not supported on image-based OS %q", info.ID) default: return "", fmt.Errorf("unsupported OS: %s", info.ID) } diff --git a/aks-node-controller/hotfix_test.go b/aks-node-controller/hotfix_test.go index 3e775354831..37b5d438bc8 100644 --- a/aks-node-controller/hotfix_test.go +++ b/aks-node-controller/hotfix_test.go @@ -175,18 +175,6 @@ func TestDetectPackageManager(t *testing.T) { assert.Contains(t, err.Error(), "unsupported OS") }) - t.Run("image-based OS reports self-update unsupported", func(t *testing.T) { - for _, id := range []string{"azurecontainerlinux", "flatcar"} { - path := filepath.Join(t.TempDir(), "os-release") - require.NoError(t, os.WriteFile(path, []byte("ID="+id+"\n"), 0644)) - a := &App{osReleasePath: path} - _, err := a.detectPackageManager() - require.Error(t, err) - assert.Contains(t, err.Error(), "not supported on image-based OS") - assert.Contains(t, err.Error(), id) - } - }) - t.Run("ACL azurelinux variant reports self-update unsupported", func(t *testing.T) { path := filepath.Join(t.TempDir(), "os-release") require.NoError(t, os.WriteFile( diff --git a/aks-node-controller/nodecustomdata_test.go b/aks-node-controller/nodecustomdata_test.go index bd050423cfa..46bdbffae01 100644 --- a/aks-node-controller/nodecustomdata_test.go +++ b/aks-node-controller/nodecustomdata_test.go @@ -8,7 +8,6 @@ import ( "fmt" "os" "path/filepath" - "runtime" "testing" "github.com/stretchr/testify/assert" @@ -54,35 +53,6 @@ write_files: assert.Equal(t, "gzip-content", string(gzipContent)) } -func TestApplyNodeCustomDataPreservesLegacyDefaultsAndCreatesParents(t *testing.T) { - tempDir := t.TempDir() - destination := filepath.Join(tempDir, "missing", "parent", "payload.txt") - renderedPath := filepath.Join(tempDir, "nodecustomdata.yml") - content := base64.StdEncoding.EncodeToString([]byte("base64-content")) - rendered := fmt.Sprintf(`write_files: -- path: %s - owner: root - encoding: base64 - content: %s -`, destination, content) - require.NoError(t, os.WriteFile(renderedPath, []byte(rendered), 0o600)) - - require.NoError(t, applyNodeCustomData(renderedPath)) - - actual, err := os.ReadFile(destination) - require.NoError(t, err) - assert.Equal(t, []byte("base64-content"), actual) - if runtime.GOOS != "windows" { - info, statErr := os.Stat(destination) - require.NoError(t, statErr) - assert.Equal(t, os.FileMode(0o644), info.Mode().Perm()) - } -} - -func TestApplyNodeCustomDataMissingFileIsNoOp(t *testing.T) { - require.NoError(t, applyNodeCustomData(filepath.Join(t.TempDir(), "missing.yml"))) -} - // TestDownloadHotfixAppliesRenderedWriteFilesWhenScriptsVersionMatches verifies that // downloadHotfix applies the rendered nodecustomdata write_files when the hotfix config's // scripts_version targets the current ANC version's YYYYMM.DD base with a strictly higher patch. diff --git a/e2e/vmss.go b/e2e/vmss.go index d133ef82d2b..17ade5e5f3d 100644 --- a/e2e/vmss.go +++ b/e2e/vmss.go @@ -137,11 +137,8 @@ func writeScriptHotfixFixture(buildDir string, fixture ScriptHotfixFixture) erro return fmt.Errorf("invalid script-hotfix fixture mode %q", fixture.Mode) } validPlatforms := map[string]bool{ - "ubuntu": true, - "mariner": true, - "azlosguard": true, - "flatcar": true, - "acl": true, + "ubuntu": true, + "mariner": true, } if !validPlatforms[fixture.Platform] { return fmt.Errorf("invalid script-hotfix fixture platform %q", fixture.Platform) diff --git a/hotfix/hotfix_generate.py b/hotfix/hotfix_generate.py index 3bf4d1dccd4..5b5c6183880 100644 --- a/hotfix/hotfix_generate.py +++ b/hotfix/hotfix_generate.py @@ -239,11 +239,6 @@ def detect_changed_varkeys(base_ref, available_varkeys=None): print(f" Skipping unsupported embedded hotfix distro: {local_path}") continue if local_path in SOURCE_TO_VARKEY: - source_path = os.path.join(ARTIFACTS_DIR, local_path) - if not os.path.isfile(source_path): - raise GenerationError( - f"changed hotfix source {local_path} does not exist at {source_path}" - ) varkey = SOURCE_TO_VARKEY[local_path] if available_varkeys is not None and varkey not in available_varkeys: raise GenerationError( @@ -387,10 +382,8 @@ def build_hotfix_template(target_varkeys, traditional_lines): if varkeys & target_varkeys: selected_blocks.append(block_lines) - if target_varkeys and not selected_blocks: - raise GenerationError("no matching write_files blocks found") if not selected_blocks: - return "#cloud-config\nwrite_files: []\n" + raise GenerationError("no matching write_files blocks found") rendered = ["#cloud-config\n", "write_files:\n"] for block_lines in selected_blocks: diff --git a/hotfix/hotfix_generate_test.py b/hotfix/hotfix_generate_test.py index 17c7c28db8e..79b1df48fb9 100644 --- a/hotfix/hotfix_generate_test.py +++ b/hotfix/hotfix_generate_test.py @@ -99,14 +99,6 @@ def test_build_hotfix_template_selects_only_requested_blocks(self): self.assertIn("provisionSource", rendered) self.assertNotIn("provisionSourceUbuntu", rendered) - def test_build_hotfix_template_emits_valid_empty_document(self): - rendered = hotfix_generate.build_hotfix_template( - set(), - TRADITIONAL_TEMPLATE.splitlines(keepends=True), - ) - - self.assertEqual("#cloud-config\nwrite_files: []\n", rendered) - def test_detect_changed_varkeys_expands_distro_group(self): with tempfile.TemporaryDirectory() as temp_dir: artifacts = Path(temp_dir) From aa1852e50c584fd81674468b23ff19800cf1d031 Mon Sep 17 00:00:00 2001 From: Devin Wong Date: Wed, 9 Sep 2026 15:27:44 -0700 Subject: [PATCH 19/26] refactor(anc): drop legacy Mariner OS detection Keep Azure Linux using the existing mariner payload filename while rejecting the retired ID=mariner OS. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4ae1dd11-ec06-4c26-bfc8-d1ef1f89fcf9 --- aks-node-controller/README.md | 5 +++-- aks-node-controller/embeddednodecustomdata.go | 2 +- aks-node-controller/embeddednodecustomdata_test.go | 4 ++-- aks-node-controller/hotfix.go | 4 ++-- aks-node-controller/hotfix_test.go | 10 +++++++++- 5 files changed, 17 insertions(+), 8 deletions(-) diff --git a/aks-node-controller/README.md b/aks-node-controller/README.md index 943f0f3c68e..67b0d9ff7bc 100644 --- a/aks-node-controller/README.md +++ b/aks-node-controller/README.md @@ -164,8 +164,9 @@ updates to the ANC binary itself. The generated files live under `aks-node-controller/scripthotfix/generated/` as `rendered_nodecustomdata_.yml`. The generator selects only changed hotfixable entries from `nodecustomdata.yml`, then renders only Ubuntu and -Mariner/standard Azure Linux variants through AgentBaker's production Go-template -functions. OS Guard, ACL, and Flatcar are explicitly skipped during embedded +standard Azure Linux variants through AgentBaker's production Go-template +functions. Azure Linux retains the `mariner` payload filename; the legacy +`ID=mariner` OS is no longer supported. OS Guard, ACL, and Flatcar are explicitly skipped during embedded application, including variants that share the `azurelinux` OS ID. Their distro-specific source changes do not trigger payload generation. diff --git a/aks-node-controller/embeddednodecustomdata.go b/aks-node-controller/embeddednodecustomdata.go index 8532a9702a6..d1068f57532 100644 --- a/aks-node-controller/embeddednodecustomdata.go +++ b/aks-node-controller/embeddednodecustomdata.go @@ -93,7 +93,7 @@ func classifyNodeCustomDataPlatform(osReleasePath string) (nodeCustomDataPlatfor return nodeCustomDataPlatformUnsupported, nil case id == "ubuntu": return nodeCustomDataPlatformUbuntu, nil - case id == "mariner", id == osReleaseIDAzureLinux: + case id == osReleaseIDAzureLinux: return nodeCustomDataPlatformMariner, nil case id == "": return "", fmt.Errorf("ID is missing from %s", osReleasePath) diff --git a/aks-node-controller/embeddednodecustomdata_test.go b/aks-node-controller/embeddednodecustomdata_test.go index 0ea4edcf8d0..e8c70676efc 100644 --- a/aks-node-controller/embeddednodecustomdata_test.go +++ b/aks-node-controller/embeddednodecustomdata_test.go @@ -20,7 +20,6 @@ func TestClassifyNodeCustomDataPlatform(t *testing.T) { expected nodeCustomDataPlatform }{ {"Ubuntu", "ID=ubuntu\n", nodeCustomDataPlatformUbuntu}, - {"Mariner", "ID=mariner\n", nodeCustomDataPlatformMariner}, {"Azure Linux", "ID=azurelinux\n", nodeCustomDataPlatformMariner}, {"OS Guard", "ID=azurelinux\nVARIANT_ID=osguard\n", nodeCustomDataPlatformUnsupported}, {"ACL variant", "ID=azurelinux\nVARIANT_ID=azurecontainerlinux\n", nodeCustomDataPlatformUnsupported}, @@ -47,7 +46,7 @@ func TestClassifyNodeCustomDataPlatform(t *testing.T) { } func TestApplyEmbeddedNodeCustomData(t *testing.T) { - for _, id := range []string{"ubuntu", "mariner", "azurelinux"} { + for _, id := range []string{"ubuntu", "azurelinux"} { t.Run(id, func(t *testing.T) { directory := t.TempDir() t.Setenv("TMPDIR", directory) @@ -102,6 +101,7 @@ func TestApplyEmbeddedNodeCustomDataErrorsAndCleanup(t *testing.T) { {name: "missing active", missing: "active", wantError: "read embedded hotfix state"}, {name: "missing release", active: "true", missing: "release", wantError: "read OS release"}, {name: "unknown OS", active: "true", release: "ID=other", wantError: "unsupported OS ID"}, + {name: "legacy mariner", active: "true", release: "ID=mariner", wantError: "unsupported OS ID"}, {name: "missing ID", active: "true", release: "VERSION_ID=3.0", wantError: "ID is missing"}, {name: "missing payload", active: "true", release: "ID=ubuntu", missing: "payload", wantError: "read embedded nodecustomdata"}, {name: "malformed YAML", active: "true", release: "ID=ubuntu", payload: "write_files: [", wantError: "unmarshal nodecustomdata"}, diff --git a/aks-node-controller/hotfix.go b/aks-node-controller/hotfix.go index 6291c854ad0..05292cb7415 100644 --- a/aks-node-controller/hotfix.go +++ b/aks-node-controller/hotfix.go @@ -221,7 +221,7 @@ func readHotfixConfig(path string) (*hotfixConfig, error) { // platformInfo holds the OS family, platform identity, and architecture for the current host. type platformInfo struct { OS string // e.g. "linux", "windows" - ID string // e.g. "ubuntu", "azurelinux", "mariner" + ID string // e.g. "ubuntu", "azurelinux" VariantID string // e.g. "azurecontainerlinux", "osguard" VersionID string // e.g. "22.04", "3.0" Arch string // e.g. "amd64", "arm64" @@ -282,7 +282,7 @@ func (a *App) detectPackageManager() (packageManager, error) { switch info.ID { case "ubuntu": return pkgMgrApt, nil - case osReleaseIDAzureLinux, "mariner": + case osReleaseIDAzureLinux: return preferredRpmManager(), nil default: return "", fmt.Errorf("unsupported OS: %s", info.ID) diff --git a/aks-node-controller/hotfix_test.go b/aks-node-controller/hotfix_test.go index 37b5d438bc8..9accd4185ae 100644 --- a/aks-node-controller/hotfix_test.go +++ b/aks-node-controller/hotfix_test.go @@ -157,7 +157,7 @@ func TestDetectPackageManager(t *testing.T) { assert.Equal(t, pkgMgrApt, pkgMgr) }) - t.Run("mariner or azurelinux returns dnf or tdnf", func(t *testing.T) { + t.Run("azurelinux returns dnf or tdnf", func(t *testing.T) { path := filepath.Join(t.TempDir(), "os-release") require.NoError(t, os.WriteFile(path, []byte(`ID="azurelinux"`+"\n"), 0644)) a := &App{osReleasePath: path} @@ -175,6 +175,14 @@ func TestDetectPackageManager(t *testing.T) { assert.Contains(t, err.Error(), "unsupported OS") }) + t.Run("legacy mariner is unsupported", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "os-release") + require.NoError(t, os.WriteFile(path, []byte("ID=mariner\n"), 0644)) + a := &App{osReleasePath: path} + _, err := a.detectPackageManager() + require.ErrorContains(t, err, "unsupported OS: mariner") + }) + t.Run("ACL azurelinux variant reports self-update unsupported", func(t *testing.T) { path := filepath.Join(t.TempDir(), "os-release") require.NoError(t, os.WriteFile( From 96e6eed64c542a5e79a71918ba566aaaeb79ba64 Mon Sep 17 00:00:00 2001 From: Devin Wong Date: Thu, 10 Sep 2026 10:09:33 -0700 Subject: [PATCH 20/26] fix(hotfix): exclude custom-image provisioning wrapper Reject cse_start.sh changes during embedded payload generation until wrapper eligibility is available. Document a future aks-rp feature-flag channel and cover wrapper-only and mixed changes. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4ae1dd11-ec06-4c26-bfc8-d1ef1f89fcf9 --- aks-node-controller/README.md | 5 +++++ hotfix/hotfix_generate.py | 15 +++++++++++++-- hotfix/hotfix_generate_test.py | 24 ++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/aks-node-controller/README.md b/aks-node-controller/README.md index 67b0d9ff7bc..563e4efff2a 100644 --- a/aks-node-controller/README.md +++ b/aks-node-controller/README.md @@ -184,6 +184,11 @@ fails, earlier writes remain. Generation selects by distro only; hotfix authors must separately account for non-distro template conditions such as custom-image exclusions. +`cse_start.sh` (`provision_start.sh` on the node) is excluded from embedded +hotfixes to preserve custom-image wrappers. Generation fails explicitly if this +script differs from the VHD baseline, even when other scripts also changed. +Wrapper fixes require a new node image until runtime eligibility is available. + Script hotfix delivery is package-only. The existing base-to-version hotfix map selects the ANC package for the node's baked `YYYYMM.DD` version base; the package contains its corresponding rendered scripts. If the package cannot be installed, diff --git a/hotfix/hotfix_generate.py b/hotfix/hotfix_generate.py index 5b5c6183880..cead5158040 100644 --- a/hotfix/hotfix_generate.py +++ b/hotfix/hotfix_generate.py @@ -62,9 +62,8 @@ "mariner/cse_install_mariner.sh": "provisionInstallsMariner", # CSE config "cse_config.sh": "provisionConfigs", - # CSE main / start + # CSE main "cse_main.sh": "provisionScript", - "cse_start.sh": "provisionStartScript", # Other scripts present in traditional nodecustomdata "configure-azure-network.sh": "configureAzureNetworkScript", "init-aks-cloud.sh": "initAKSCloud", @@ -238,6 +237,18 @@ def detect_changed_varkeys(base_ref, available_varkeys=None): if local_path.startswith(UNSUPPORTED_DISTRO_DIRS): print(f" Skipping unsupported embedded hotfix distro: {local_path}") continue + if local_path == "cse_start.sh": + # Custom images may supply their own provision_start.sh; distro-only + # rendering loses the template's not IsCustomImage condition. + # Future option: aks-rp can send explicit wrapper-hotfix eligibility + # via enabled_features.sh. The launcher already exports those flags; + # ANC would omit this entry unless explicitly allowed, including when + # the flag is absent, before calling the existing applyNodeCustomData. + raise GenerationError( + "cse_start.sh cannot be delivered as an embedded hotfix because " + "custom-image wrapper eligibility is unavailable; publish a new " + "node image or implement explicit runtime eligibility" + ) if local_path in SOURCE_TO_VARKEY: varkey = SOURCE_TO_VARKEY[local_path] if available_varkeys is not None and varkey not in available_varkeys: diff --git a/hotfix/hotfix_generate_test.py b/hotfix/hotfix_generate_test.py index 79b1df48fb9..2c6f793b85e 100644 --- a/hotfix/hotfix_generate_test.py +++ b/hotfix/hotfix_generate_test.py @@ -256,6 +256,30 @@ def test_write_hotfix_file_without_version_keeps_missing_target_absent(self): hotfix_generate.write_hotfix_file("") self.assertFalse(target.exists()) + def test_cse_start_hotfix_fails_even_with_supported_changes(self): + self.assertNotIn("cse_start.sh", hotfix_generate.SOURCE_TO_VARKEY) + for sources in (["cse_start.sh"], ["cse_config.sh", "cse_start.sh"]): + with self.subTest(sources=sources): + result = subprocess.CompletedProcess( + args=[], + returncode=0, + stdout="".join( + f"{hotfix_generate.ARTIFACTS_DIR}/{source}\n" + for source in sources + ), + ) + with mock.patch.object( + hotfix_generate.subprocess, "run", return_value=result + ): + with self.assertRaisesRegex( + hotfix_generate.GenerationError, + "cse_start.sh cannot be delivered.*custom-image wrapper eligibility", + ): + hotfix_generate.detect_changed_varkeys( + "base", + available_varkeys={"provisionConfigs", "provisionStartScript"}, + ) + def test_unmapped_hotfixable_script_fails(self): with tempfile.TemporaryDirectory() as temp_dir: changed = Path(temp_dir) / "unmapped.sh" From e0127f493653ba909052090c3ca2a050680d0f4a Mon Sep 17 00:00:00 2001 From: Devin Wong Date: Thu, 10 Sep 2026 14:35:18 -0700 Subject: [PATCH 21/26] Simplify embedded hotfix activation to payload presence Remove the active marker and empty distro payloads. Skip absent distro YAMLs, retain other read errors, and keep a README placeholder for go:embed. Update generation, fixtures, and documentation. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4ae1dd11-ec06-4c26-bfc8-d1ef1f89fcf9 --- aks-node-controller/README.md | 7 ++-- aks-node-controller/app.go | 2 +- aks-node-controller/embeddednodecustomdata.go | 13 +++--- .../embeddednodecustomdata_test.go | 42 +++++++++---------- .../scripthotfix/generated/README | 3 ++ .../scripthotfix/generated/active | 1 - .../rendered_nodecustomdata_mariner.yml | 2 - .../rendered_nodecustomdata_ubuntu.yml | 2 - e2e/scenario/vmss.go | 3 -- e2e/scenario/vmss_test.go | 4 ++ hotfix/hotfix_generate.py | 2 - hotfix/hotfix_generate_test.py | 29 +++++++++++-- 12 files changed, 63 insertions(+), 47 deletions(-) create mode 100644 aks-node-controller/scripthotfix/generated/README delete mode 100644 aks-node-controller/scripthotfix/generated/active delete mode 100644 aks-node-controller/scripthotfix/generated/rendered_nodecustomdata_mariner.yml delete mode 100644 aks-node-controller/scripthotfix/generated/rendered_nodecustomdata_ubuntu.yml diff --git a/aks-node-controller/README.md b/aks-node-controller/README.md index 563e4efff2a..96736fee842 100644 --- a/aks-node-controller/README.md +++ b/aks-node-controller/README.md @@ -170,9 +170,10 @@ functions. Azure Linux retains the `mariner` payload filename; the legacy application, including variants that share the `azurelinux` OS ID. Their distro-specific source changes do not trigger payload generation. -The repository keeps two empty YAML templates (`write_files: []`) and an -`active=false` marker until a script hotfix is generated. Generation populates -those two payloads and sets `active=true`. +The repository keeps a README placeholder so `go:embed` builds without any +script hotfix payloads. Generation replaces it with Ubuntu and Azure Linux YAMLs. +ANC skips application when the local platform's YAML is absent; no separate +activation flag is needed. Other payload read errors are logged. When a PR has no new script hotfix, generation leaves the existing rendered payload unchanged. The active ANC version pointer is likewise retained until it diff --git a/aks-node-controller/app.go b/aks-node-controller/app.go index 5b765e5e1c6..e174473c9e4 100644 --- a/aks-node-controller/app.go +++ b/aks-node-controller/app.go @@ -709,7 +709,7 @@ func (a *App) runProvision(ctx context.Context, flags ProvisionFlags, dryRun boo applyHotfix := a.applyEmbeddedHotfix if applyHotfix == nil { applyHotfix = func(osReleasePath string) error { - return applyEmbeddedNodeCustomDataIfActive(embeddedGeneratedNodeCustomData, osReleasePath) + return applyEmbeddedNodeCustomData(embeddedGeneratedNodeCustomData, osReleasePath) } } if err := applyHotfix(a.osReleasePath); err != nil { diff --git a/aks-node-controller/embeddednodecustomdata.go b/aks-node-controller/embeddednodecustomdata.go index d1068f57532..91a4e320f43 100644 --- a/aks-node-controller/embeddednodecustomdata.go +++ b/aks-node-controller/embeddednodecustomdata.go @@ -2,6 +2,7 @@ package main import ( "embed" + "errors" "fmt" "io/fs" "log/slog" @@ -29,14 +30,7 @@ const ( //go:embed scripthotfix/generated var embeddedGeneratedNodeCustomData embed.FS -func applyEmbeddedNodeCustomDataIfActive(payloadFS fs.FS, osReleasePath string) error { - active, err := fs.ReadFile(payloadFS, "scripthotfix/generated/active") - if err != nil { - return fmt.Errorf("read embedded hotfix state: %w", err) - } - if strings.TrimSpace(string(active)) != "true" { - return nil - } +func applyEmbeddedNodeCustomData(payloadFS fs.FS, osReleasePath string) error { if osReleasePath == "" { osReleasePath = defaultOSReleasePath } @@ -50,6 +44,9 @@ func applyEmbeddedNodeCustomDataIfActive(payloadFS fs.FS, osReleasePath string) } renderedPath := fmt.Sprintf("scripthotfix/generated/rendered_nodecustomdata_%s.yml", platform) data, err := fs.ReadFile(payloadFS, renderedPath) + if errors.Is(err, fs.ErrNotExist) { + return nil + } if err != nil { return fmt.Errorf("read embedded nodecustomdata %s: %w", renderedPath, err) } diff --git a/aks-node-controller/embeddednodecustomdata_test.go b/aks-node-controller/embeddednodecustomdata_test.go index e8c70676efc..1dacafb4d6e 100644 --- a/aks-node-controller/embeddednodecustomdata_test.go +++ b/aks-node-controller/embeddednodecustomdata_test.go @@ -2,6 +2,7 @@ package main import ( "encoding/base64" + "io/fs" "os" "path/filepath" "runtime" @@ -35,10 +36,7 @@ func TestClassifyNodeCustomDataPlatform(t *testing.T) { require.NoError(t, err) assert.Equal(t, test.expected, actual) if actual == nodeCustomDataPlatformUnsupported { - files := fstest.MapFS{ - "scripthotfix/generated/active": &fstest.MapFile{Data: []byte("true\n")}, - } - require.NoError(t, applyEmbeddedNodeCustomDataIfActive(files, releasePath), + require.NoError(t, applyEmbeddedNodeCustomData(fstest.MapFS{}, releasePath), "unsupported platforms must skip without reading any payload") } }) @@ -67,10 +65,9 @@ func TestApplyEmbeddedNodeCustomData(t *testing.T) { platform = string(nodeCustomDataPlatformMariner) } files := fstest.MapFS{ - "scripthotfix/generated/active": &fstest.MapFile{Data: []byte("true\n")}, "scripthotfix/generated/rendered_nodecustomdata_" + platform + ".yml": &fstest.MapFile{Data: data}, } - require.NoError(t, applyEmbeddedNodeCustomDataIfActive(files, releasePath)) + require.NoError(t, applyEmbeddedNodeCustomData(files, releasePath)) for _, destination := range []string{existing, missing} { actual, readErr := os.ReadFile(destination) require.NoError(t, readErr) @@ -91,23 +88,22 @@ func TestApplyEmbeddedNodeCustomData(t *testing.T) { func TestApplyEmbeddedNodeCustomDataErrorsAndCleanup(t *testing.T) { tests := []struct { name string - active string release string payload string missing string wantError string }{ - {name: "inactive skips missing OS release", active: "false\n", missing: "release"}, - {name: "missing active", missing: "active", wantError: "read embedded hotfix state"}, - {name: "missing release", active: "true", missing: "release", wantError: "read OS release"}, - {name: "unknown OS", active: "true", release: "ID=other", wantError: "unsupported OS ID"}, - {name: "legacy mariner", active: "true", release: "ID=mariner", wantError: "unsupported OS ID"}, - {name: "missing ID", active: "true", release: "VERSION_ID=3.0", wantError: "ID is missing"}, - {name: "missing payload", active: "true", release: "ID=ubuntu", missing: "payload", wantError: "read embedded nodecustomdata"}, - {name: "malformed YAML", active: "true", release: "ID=ubuntu", payload: "write_files: [", wantError: "unmarshal nodecustomdata"}, - {name: "invalid entry", active: "true", release: "ID=ubuntu", payload: "write_files:\n- content: invalid\n", wantError: "path is required"}, - {name: "empty payload", active: "true", release: "ID=ubuntu", payload: "write_files: []\n"}, - {name: "temporary directory unavailable", active: "true", release: "ID=ubuntu", payload: "write_files: []\n", missing: "temp", wantError: "create temporary nodecustomdata"}, + {name: "missing release", missing: "release", wantError: "read OS release"}, + {name: "unknown OS", release: "ID=other", wantError: "unsupported OS ID"}, + {name: "legacy mariner", release: "ID=mariner", wantError: "unsupported OS ID"}, + {name: "missing ID", release: "VERSION_ID=3.0", wantError: "ID is missing"}, + {name: "missing payload", release: "ID=ubuntu", missing: "payload"}, + {name: "other platform payload only", release: "ID=azurelinux", payload: "write_files: ["}, + {name: "unreadable payload directory", release: "ID=ubuntu", missing: "payload-file", wantError: "read embedded nodecustomdata"}, + {name: "malformed YAML", release: "ID=ubuntu", payload: "write_files: [", wantError: "unmarshal nodecustomdata"}, + {name: "invalid entry", release: "ID=ubuntu", payload: "write_files:\n- content: invalid\n", wantError: "path is required"}, + {name: "empty payload", release: "ID=ubuntu", payload: "write_files: []\n"}, + {name: "temporary directory unavailable", release: "ID=ubuntu", payload: "write_files: []\n", missing: "temp", wantError: "create temporary nodecustomdata"}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { @@ -122,14 +118,16 @@ func TestApplyEmbeddedNodeCustomDataErrorsAndCleanup(t *testing.T) { if test.missing != "release" { require.NoError(t, os.WriteFile(releasePath, []byte(test.release), 0o600)) } - files := fstest.MapFS{} - if test.missing != "active" { - files["scripthotfix/generated/active"] = &fstest.MapFile{Data: []byte(test.active)} + files := fstest.MapFS{ + "scripthotfix/generated/README": &fstest.MapFile{Data: []byte("placeholder")}, } if test.missing != "payload" { files["scripthotfix/generated/rendered_nodecustomdata_ubuntu.yml"] = &fstest.MapFile{Data: []byte(test.payload)} } - err := applyEmbeddedNodeCustomDataIfActive(files, releasePath) + if test.missing == "payload-file" { + files["scripthotfix/generated/rendered_nodecustomdata_ubuntu.yml"].Mode = fs.ModeDir + } + err := applyEmbeddedNodeCustomData(files, releasePath) if test.wantError == "" { require.NoError(t, err) } else { diff --git a/aks-node-controller/scripthotfix/generated/README b/aks-node-controller/scripthotfix/generated/README new file mode 100644 index 00000000000..994aba0673b --- /dev/null +++ b/aks-node-controller/scripthotfix/generated/README @@ -0,0 +1,3 @@ +This placeholder keeps go:embed valid when no script hotfix payloads exist. +hotfix/hotfix_generate.py replaces this directory with rendered distro YAMLs. +An absent rendered_nodecustomdata_.yml means no hotfix for that platform. diff --git a/aks-node-controller/scripthotfix/generated/active b/aks-node-controller/scripthotfix/generated/active deleted file mode 100644 index c508d5366f7..00000000000 --- a/aks-node-controller/scripthotfix/generated/active +++ /dev/null @@ -1 +0,0 @@ -false diff --git a/aks-node-controller/scripthotfix/generated/rendered_nodecustomdata_mariner.yml b/aks-node-controller/scripthotfix/generated/rendered_nodecustomdata_mariner.yml deleted file mode 100644 index 7028abd713c..00000000000 --- a/aks-node-controller/scripthotfix/generated/rendered_nodecustomdata_mariner.yml +++ /dev/null @@ -1,2 +0,0 @@ -#cloud-config -write_files: [] diff --git a/aks-node-controller/scripthotfix/generated/rendered_nodecustomdata_ubuntu.yml b/aks-node-controller/scripthotfix/generated/rendered_nodecustomdata_ubuntu.yml deleted file mode 100644 index 7028abd713c..00000000000 --- a/aks-node-controller/scripthotfix/generated/rendered_nodecustomdata_ubuntu.yml +++ /dev/null @@ -1,2 +0,0 @@ -#cloud-config -write_files: [] diff --git a/e2e/scenario/vmss.go b/e2e/scenario/vmss.go index c0575ebd01c..58e538117e9 100644 --- a/e2e/scenario/vmss.go +++ b/e2e/scenario/vmss.go @@ -176,9 +176,6 @@ func writeScriptHotfixFixture(buildDir string, fixture ScriptHotfixFixture) erro if err := os.WriteFile(outputPath, data, 0o600); err != nil { return fmt.Errorf("write rendered script-hotfix fixture: %w", err) } - if err := os.WriteFile(filepath.Join(generatedDir, "active"), []byte("true\n"), 0o600); err != nil { - return fmt.Errorf("enable rendered script-hotfix fixture: %w", err) - } return nil } diff --git a/e2e/scenario/vmss_test.go b/e2e/scenario/vmss_test.go index 9640965c474..98c4a6f3aa5 100644 --- a/e2e/scenario/vmss_test.go +++ b/e2e/scenario/vmss_test.go @@ -24,6 +24,10 @@ func TestWriteScriptHotfixFixture(t *testing.T) { } require.NoError(t, writeScriptHotfixFixture(buildDir, fixture)) + entries, err := os.ReadDir(filepath.Join(buildDir, "scripthotfix", "generated")) + require.NoError(t, err) + require.Len(t, entries, 1) + require.Equal(t, "rendered_nodecustomdata_ubuntu.yml", entries[0].Name()) renderedData, err := os.ReadFile(filepath.Join( buildDir, diff --git a/hotfix/hotfix_generate.py b/hotfix/hotfix_generate.py index cead5158040..9ec39f76ac9 100644 --- a/hotfix/hotfix_generate.py +++ b/hotfix/hotfix_generate.py @@ -437,8 +437,6 @@ def write_rendered_payload(target_varkeys, traditional_lines): except FileNotFoundError: pass - with open(os.path.join(GENERATED_DIR, "active"), "w", newline="\n") as active_file: - active_file.write("true\n") print( f"Rendered {len(target_varkeys)} hotfix variable keys into {GENERATED_DIR}", file=sys.stderr, diff --git a/hotfix/hotfix_generate_test.py b/hotfix/hotfix_generate_test.py index 2c6f793b85e..2e50dbc2e1a 100644 --- a/hotfix/hotfix_generate_test.py +++ b/hotfix/hotfix_generate_test.py @@ -155,6 +155,8 @@ def test_detect_changed_varkeys_skips_unsupported_distros(self): def test_write_rendered_payload_uses_canonical_renderer(self): with tempfile.TemporaryDirectory() as temp_dir: generated = Path(temp_dir) / "generated" + generated.mkdir() + (generated / "README").write_text("placeholder\n") def render(command, check): self.assertTrue(check) @@ -197,13 +199,35 @@ def render(command, check): self.assertIn("/opt/azure/containers/provision_source.sh", content) self.assertNotIn("{{", content) self.assertFalse((generated / ".nodecustomdata-hotfix.template").exists()) - self.assertEqual("true\n", (generated / "active").read_text()) + self.assertEqual( + {f"rendered_nodecustomdata_{platform}.yml" for platform in expected}, + {path.name for path in generated.iterdir()}, + ) + + def test_write_rendered_payload_preserves_placeholder_without_hotfix(self): + with tempfile.TemporaryDirectory() as temp_dir: + generated = Path(temp_dir) / "generated" + generated.mkdir() + placeholder = generated / "README" + placeholder.write_text("placeholder\n") + with mock.patch.object( + hotfix_generate, "GENERATED_DIR", str(generated) + ), mock.patch.object( + hotfix_generate.subprocess, "run" + ) as run: + hotfix_generate.write_rendered_payload( + set(), + TRADITIONAL_TEMPLATE.splitlines(keepends=True), + ) + + run.assert_not_called() + self.assertEqual("placeholder\n", placeholder.read_text()) + self.assertEqual({"README"}, {path.name for path in generated.iterdir()}) def test_write_rendered_payload_preserves_previous_hotfix_when_unchanged(self): with tempfile.TemporaryDirectory() as temp_dir: generated = Path(temp_dir) / "generated" generated.mkdir() - (generated / "active").write_text("true\n") platforms = ("ubuntu", "mariner") for platform in platforms: (generated / f"rendered_nodecustomdata_{platform}.yml").write_text( @@ -221,7 +245,6 @@ def test_write_rendered_payload_preserves_previous_hotfix_when_unchanged(self): ) run.assert_not_called() - self.assertEqual("true\n", (generated / "active").read_text()) for platform in platforms: self.assertIn( f"/{platform}-existing", From ed2c2245e4ec613e4c57b543f954a89eb59ce1a0 Mon Sep 17 00:00:00 2001 From: Devin Wong Date: Thu, 10 Sep 2026 14:54:08 -0700 Subject: [PATCH 22/26] Move nodecustomdata renderer tests into baker_test.go Keep renderer coverage alongside baker.go after the renderer implementation was consolidated there. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4ae1dd11-ec06-4c26-bfc8-d1ef1f89fcf9 --- pkg/agent/baker_test.go | 110 ++++++++++++++++++++++ pkg/agent/nodecustomdata_render_test.go | 117 ------------------------ 2 files changed, 110 insertions(+), 117 deletions(-) delete mode 100644 pkg/agent/nodecustomdata_render_test.go diff --git a/pkg/agent/baker_test.go b/pkg/agent/baker_test.go index 7f5fca2538c..5ea33504f4b 100644 --- a/pkg/agent/baker_test.go +++ b/pkg/agent/baker_test.go @@ -14,6 +14,7 @@ import ( "path/filepath" "regexp" "strings" + "testing" "github.com/Azure/agentbaker/parts" "github.com/Azure/agentbaker/pkg/agent/datamodel" @@ -23,6 +24,7 @@ import ( flatcar1_1 "github.com/coreos/butane/config/flatcar/v1_1" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/stretchr/testify/require" "github.com/vincent-petithory/dataurl" ) @@ -45,6 +47,114 @@ health-check.localdns.local:53 { # KubeDNS overrides apply to DNS traffic from pods with dnsPolicy:ClusterFirst (referred to as KubeDNS traffic). ` +func TestRenderLinuxNodeCustomDataTemplateUsesBakerPlatformFunctions(t *testing.T) { + template := []byte(`#cloud-config +write_files: +{{if IsACL}} +- path: /acl +{{else if IsAzlOSGuard}} +- path: /azlosguard +{{else if IsMariner}} +- path: /mariner +{{else if IsFlatcar}} +- path: /flatcar +{{else}} +- path: /ubuntu +{{end}} +`) + tests := []struct { + name string + distro datamodel.Distro + expected string + }{ + {name: "Ubuntu", distro: datamodel.AKSUbuntuContainerd2204Gen2, expected: "/ubuntu"}, + {name: "Mariner", distro: datamodel.AKSAzureLinuxV3Gen2, expected: "/mariner"}, + {name: "ACL", distro: datamodel.AKSACLGen2TL, expected: "/acl"}, + {name: "OS Guard", distro: datamodel.AKSAzureLinuxV3OSGuardGen2FIPSTL, expected: "/azlosguard"}, + {name: "Flatcar", distro: datamodel.AKSFlatcarGen2, expected: "/flatcar"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + rendered, err := RenderLinuxNodeCustomDataTemplate( + template, + newNodeCustomDataRenderConfig(test.distro), + ) + + require.NoError(t, err) + require.Contains(t, rendered, "- path: "+test.expected) + require.False(t, strings.Contains(rendered, "{{")) + }) + } +} + +func TestRenderLinuxNodeCustomDataTemplateRejectsMissingDependencies(t *testing.T) { + tests := []struct { + name string + remove func(*datamodel.NodeBootstrappingConfiguration) + }{ + { + name: "orchestrator profile", + remove: func(config *datamodel.NodeBootstrappingConfiguration) { + config.ContainerService.Properties.OrchestratorProfile = nil + }, + }, + { + name: "Kubernetes components", + remove: func(config *datamodel.NodeBootstrappingConfiguration) { + config.K8sComponents = nil + }, + }, + { + name: "cloud spec config", + remove: func(config *datamodel.NodeBootstrappingConfiguration) { + config.CloudSpecConfig = nil + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + config := newNodeCustomDataRenderConfig(datamodel.AKSUbuntuContainerd2204Gen2) + test.remove(config) + + _, err := RenderLinuxNodeCustomDataTemplate([]byte("#cloud-config\nwrite_files: []\n"), config) + + require.EqualError(t, err, "node bootstrapping configuration is incomplete") + }) + } +} + +func newNodeCustomDataRenderConfig(distro datamodel.Distro) *datamodel.NodeBootstrappingConfiguration { + profile := &datamodel.AgentPoolProfile{ + Name: "hotfix-render-test", + OSType: datamodel.Linux, + Distro: distro, + } + return &datamodel.NodeBootstrappingConfiguration{ + ContainerService: &datamodel.ContainerService{ + Location: "eastus", + Properties: &datamodel.Properties{ + OrchestratorProfile: &datamodel.OrchestratorProfile{ + OrchestratorVersion: "1.29.0", + OrchestratorType: datamodel.Kubernetes, + KubernetesConfig: &datamodel.KubernetesConfig{ + ContainerRuntimeConfig: map[string]string{}, + }, + }, + HostedMasterProfile: &datamodel.HostedMasterProfile{ + FQDN: "hotfix-render.invalid", + }, + AgentPoolProfiles: []*datamodel.AgentPoolProfile{profile}, + }, + }, + AgentPoolProfile: profile, + CloudSpecConfig: datamodel.AzurePublicCloudSpecForTest, + K8sComponents: &datamodel.K8sComponents{}, + KubeletConfig: map[string]string{}, + } +} + type decodedValue struct { value string mode int64 diff --git a/pkg/agent/nodecustomdata_render_test.go b/pkg/agent/nodecustomdata_render_test.go deleted file mode 100644 index 3417f7f82b7..00000000000 --- a/pkg/agent/nodecustomdata_render_test.go +++ /dev/null @@ -1,117 +0,0 @@ -package agent - -import ( - "strings" - "testing" - - "github.com/Azure/agentbaker/pkg/agent/datamodel" - "github.com/stretchr/testify/require" -) - -func TestRenderLinuxNodeCustomDataTemplateUsesBakerPlatformFunctions(t *testing.T) { - template := []byte(`#cloud-config -write_files: -{{if IsACL}} -- path: /acl -{{else if IsAzlOSGuard}} -- path: /azlosguard -{{else if IsMariner}} -- path: /mariner -{{else if IsFlatcar}} -- path: /flatcar -{{else}} -- path: /ubuntu -{{end}} -`) - tests := []struct { - name string - distro datamodel.Distro - expected string - }{ - {name: "Ubuntu", distro: datamodel.AKSUbuntuContainerd2204Gen2, expected: "/ubuntu"}, - {name: "Mariner", distro: datamodel.AKSAzureLinuxV3Gen2, expected: "/mariner"}, - {name: "ACL", distro: datamodel.AKSACLGen2TL, expected: "/acl"}, - {name: "OS Guard", distro: datamodel.AKSAzureLinuxV3OSGuardGen2FIPSTL, expected: "/azlosguard"}, - {name: "Flatcar", distro: datamodel.AKSFlatcarGen2, expected: "/flatcar"}, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - rendered, err := RenderLinuxNodeCustomDataTemplate( - template, - newNodeCustomDataRenderConfig(test.distro), - ) - - require.NoError(t, err) - require.Contains(t, rendered, "- path: "+test.expected) - require.False(t, strings.Contains(rendered, "{{")) - }) - } -} - -func TestRenderLinuxNodeCustomDataTemplateRejectsMissingDependencies(t *testing.T) { - tests := []struct { - name string - remove func(*datamodel.NodeBootstrappingConfiguration) - }{ - { - name: "orchestrator profile", - remove: func(config *datamodel.NodeBootstrappingConfiguration) { - config.ContainerService.Properties.OrchestratorProfile = nil - }, - }, - { - name: "Kubernetes components", - remove: func(config *datamodel.NodeBootstrappingConfiguration) { - config.K8sComponents = nil - }, - }, - { - name: "cloud spec config", - remove: func(config *datamodel.NodeBootstrappingConfiguration) { - config.CloudSpecConfig = nil - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - config := newNodeCustomDataRenderConfig(datamodel.AKSUbuntuContainerd2204Gen2) - test.remove(config) - - _, err := RenderLinuxNodeCustomDataTemplate([]byte("#cloud-config\nwrite_files: []\n"), config) - - require.EqualError(t, err, "node bootstrapping configuration is incomplete") - }) - } -} - -func newNodeCustomDataRenderConfig(distro datamodel.Distro) *datamodel.NodeBootstrappingConfiguration { - profile := &datamodel.AgentPoolProfile{ - Name: "hotfix-render-test", - OSType: datamodel.Linux, - Distro: distro, - } - return &datamodel.NodeBootstrappingConfiguration{ - ContainerService: &datamodel.ContainerService{ - Location: "eastus", - Properties: &datamodel.Properties{ - OrchestratorProfile: &datamodel.OrchestratorProfile{ - OrchestratorVersion: "1.29.0", - OrchestratorType: datamodel.Kubernetes, - KubernetesConfig: &datamodel.KubernetesConfig{ - ContainerRuntimeConfig: map[string]string{}, - }, - }, - HostedMasterProfile: &datamodel.HostedMasterProfile{ - FQDN: "hotfix-render.invalid", - }, - AgentPoolProfiles: []*datamodel.AgentPoolProfile{profile}, - }, - }, - AgentPoolProfile: profile, - CloudSpecConfig: datamodel.AzurePublicCloudSpecForTest, - K8sComponents: &datamodel.K8sComponents{}, - KubeletConfig: map[string]string{}, - } -} From 0220ef3219bdc24b5d6d854ee1ceaabe0dd3e939 Mon Sep 17 00:00:00 2001 From: Devin Wong Date: Thu, 10 Sep 2026 15:02:10 -0700 Subject: [PATCH 23/26] Align renderer configuration assumptions with baker helpers Remove upfront configuration nil checks and their dedicated test. Document the complete-configuration prerequisite while preserving template parsing and execution error handling. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4ae1dd11-ec06-4c26-bfc8-d1ef1f89fcf9 --- pkg/agent/baker.go | 12 ++---------- pkg/agent/baker_test.go | 37 ------------------------------------- 2 files changed, 2 insertions(+), 47 deletions(-) diff --git a/pkg/agent/baker.go b/pkg/agent/baker.go index 944c6a180ad..2dde3d9f364 100644 --- a/pkg/agent/baker.go +++ b/pkg/agent/baker.go @@ -634,17 +634,9 @@ func (t *TemplateGenerator) getSingleLine(textFilename string, profile interface // RenderLinuxNodeCustomDataTemplate renders a nodecustomdata template with the // same variables and functions used by the production AgentBaker path. +// Callers must supply a complete configuration, as required by the production +// rendering helpers. func RenderLinuxNodeCustomDataTemplate(templateContent []byte, config *datamodel.NodeBootstrappingConfiguration) (string, error) { - if config == nil || - config.AgentPoolProfile == nil || - config.ContainerService == nil || - config.ContainerService.Properties == nil || - config.ContainerService.Properties.OrchestratorProfile == nil || - config.K8sComponents == nil || - config.CloudSpecConfig == nil { - return "", fmt.Errorf("node bootstrapping configuration is incomplete") - } - parameters := getParameters(config) variables := getCustomDataVariables(config) templ := template.New("nodecustomdata template"). diff --git a/pkg/agent/baker_test.go b/pkg/agent/baker_test.go index 5ea33504f4b..db4e7b65f4c 100644 --- a/pkg/agent/baker_test.go +++ b/pkg/agent/baker_test.go @@ -88,43 +88,6 @@ write_files: } } -func TestRenderLinuxNodeCustomDataTemplateRejectsMissingDependencies(t *testing.T) { - tests := []struct { - name string - remove func(*datamodel.NodeBootstrappingConfiguration) - }{ - { - name: "orchestrator profile", - remove: func(config *datamodel.NodeBootstrappingConfiguration) { - config.ContainerService.Properties.OrchestratorProfile = nil - }, - }, - { - name: "Kubernetes components", - remove: func(config *datamodel.NodeBootstrappingConfiguration) { - config.K8sComponents = nil - }, - }, - { - name: "cloud spec config", - remove: func(config *datamodel.NodeBootstrappingConfiguration) { - config.CloudSpecConfig = nil - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - config := newNodeCustomDataRenderConfig(datamodel.AKSUbuntuContainerd2204Gen2) - test.remove(config) - - _, err := RenderLinuxNodeCustomDataTemplate([]byte("#cloud-config\nwrite_files: []\n"), config) - - require.EqualError(t, err, "node bootstrapping configuration is incomplete") - }) - } -} - func newNodeCustomDataRenderConfig(distro datamodel.Distro) *datamodel.NodeBootstrappingConfiguration { profile := &datamodel.AgentPoolProfile{ Name: "hotfix-render-test", From 38bb2eb6f72d842a6073aef2e3e25c51deb6f43e Mon Sep 17 00:00:00 2001 From: Devinwong Date: Thu, 10 Sep 2026 15:20:17 -0700 Subject: [PATCH 24/26] Fix condition to include 'osguard' variant Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- aks-node-controller/hotfix.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/aks-node-controller/hotfix.go b/aks-node-controller/hotfix.go index 8fb195c1e74..420ba49360f 100644 --- a/aks-node-controller/hotfix.go +++ b/aks-node-controller/hotfix.go @@ -241,7 +241,8 @@ func (a *App) detectPackageManager() (packageManager, error) { if err != nil { return "", err } - if info.ID == osReleaseIDAzureLinux && info.VariantID == osReleaseIDAzureContainerLinux { + if info.ID == osReleaseIDAzureLinux && + (info.VariantID == osReleaseIDAzureContainerLinux || info.VariantID == "osguard") { return "", fmt.Errorf( "PMC package-based ANC self-update is not supported on image-based OS %q variant %q", info.ID, From 29044cb57897ec7b3aa0f47db63ed39cbd60675f Mon Sep 17 00:00:00 2001 From: Devin Wong Date: Thu, 10 Sep 2026 17:25:40 -0700 Subject: [PATCH 25/26] Retain embedded hotfix YAML for node debugging Write the selected payload to /opt/azure/containers/embedded-nodecustomdata.yml and reuse the existing applier without removing the YAML. Keep legacy custom data separate and cover retention, write failures, and permissions. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4ae1dd11-ec06-4c26-bfc8-d1ef1f89fcf9 --- aks-node-controller/README.md | 12 ++-- aks-node-controller/app.go | 2 +- aks-node-controller/embeddednodecustomdata.go | 27 +++----- .../embeddednodecustomdata_test.go | 69 +++++++++++++------ 4 files changed, 67 insertions(+), 43 deletions(-) diff --git a/aks-node-controller/README.md b/aks-node-controller/README.md index 96736fee842..151e40cb7a3 100644 --- a/aks-node-controller/README.md +++ b/aks-node-controller/README.md @@ -154,10 +154,14 @@ Key components: Patched ANC binaries can embed selected Linux provisioning scripts generated from `parts/linux/cloud-init/artifacts/`. At the start of `provision`, ANC writes the -rendered nodecustomdata matching the local platform to a private temporary YAML -file and calls the existing `applyNodeCustomData` function before constructing the -normal CSE command. The temporary YAML is removed afterward. Application errors -are logged and provisioning continues. +rendered nodecustomdata matching the local platform to +`/opt/azure/containers/embedded-nodecustomdata.yml` (mode `0600` on creation) +and calls the existing `applyNodeCustomData` function before constructing the +normal CSE command. This file is retained for debugging, separate from the legacy +`nodecustomdata.yml`. It contains the most recently written payload, including +when application fails; the `applied embedded hotfix payload` log confirms +successful application. If no payload is selected, any previously retained file +is left untouched. Application errors are logged and provisioning continues. The embedded nodecustomdata coordinator distinguishes these script hotfixes from updates to the ANC binary itself. The generated files live under diff --git a/aks-node-controller/app.go b/aks-node-controller/app.go index e174473c9e4..ff0ede7d3c1 100644 --- a/aks-node-controller/app.go +++ b/aks-node-controller/app.go @@ -709,7 +709,7 @@ func (a *App) runProvision(ctx context.Context, flags ProvisionFlags, dryRun boo applyHotfix := a.applyEmbeddedHotfix if applyHotfix == nil { applyHotfix = func(osReleasePath string) error { - return applyEmbeddedNodeCustomData(embeddedGeneratedNodeCustomData, osReleasePath) + return applyEmbeddedNodeCustomData(embeddedGeneratedNodeCustomData, osReleasePath, embeddedNodeCustomDataPath) } } if err := applyHotfix(a.osReleasePath); err != nil { diff --git a/aks-node-controller/embeddednodecustomdata.go b/aks-node-controller/embeddednodecustomdata.go index 91a4e320f43..fc33df49f62 100644 --- a/aks-node-controller/embeddednodecustomdata.go +++ b/aks-node-controller/embeddednodecustomdata.go @@ -7,10 +7,12 @@ import ( "io/fs" "log/slog" "os" + "path/filepath" "strings" ) const defaultOSReleasePath = "/etc/os-release" +const embeddedNodeCustomDataPath = "/opt/azure/containers/embedded-nodecustomdata.yml" // os-release ID values that appear in more than one classification path. const ( @@ -30,7 +32,7 @@ const ( //go:embed scripthotfix/generated var embeddedGeneratedNodeCustomData embed.FS -func applyEmbeddedNodeCustomData(payloadFS fs.FS, osReleasePath string) error { +func applyEmbeddedNodeCustomData(payloadFS fs.FS, osReleasePath, outputPath string) error { if osReleasePath == "" { osReleasePath = defaultOSReleasePath } @@ -50,27 +52,16 @@ func applyEmbeddedNodeCustomData(payloadFS fs.FS, osReleasePath string) error { if err != nil { return fmt.Errorf("read embedded nodecustomdata %s: %w", renderedPath, err) } - temp, err := os.CreateTemp("", "aks-node-controller-nodecustomdata-*.yml") - if err != nil { - return fmt.Errorf("create temporary nodecustomdata: %w", err) - } - defer func() { - if err := os.Remove(temp.Name()); err != nil { - slog.Warn("failed to remove temporary nodecustomdata", "path", temp.Name(), "error", err) - } - }() - _, writeErr := temp.Write(data) - closeErr := temp.Close() - if writeErr != nil { - return fmt.Errorf("write temporary nodecustomdata: %w", writeErr) + if err := os.MkdirAll(filepath.Dir(outputPath), 0o755); err != nil { + return fmt.Errorf("create embedded nodecustomdata directory: %w", err) } - if closeErr != nil { - return fmt.Errorf("close temporary nodecustomdata: %w", closeErr) + if err := os.WriteFile(outputPath, data, 0o600); err != nil { + return fmt.Errorf("write embedded nodecustomdata %s: %w", outputPath, err) } - if err := applyNodeCustomData(temp.Name()); err != nil { + if err := applyNodeCustomData(outputPath); err != nil { return err } - slog.Info("applied embedded hotfix payload", "source", renderedPath) + slog.Info("applied embedded hotfix payload", "source", renderedPath, "path", outputPath) return nil } diff --git a/aks-node-controller/embeddednodecustomdata_test.go b/aks-node-controller/embeddednodecustomdata_test.go index 1dacafb4d6e..9a2001b35d7 100644 --- a/aks-node-controller/embeddednodecustomdata_test.go +++ b/aks-node-controller/embeddednodecustomdata_test.go @@ -36,8 +36,10 @@ func TestClassifyNodeCustomDataPlatform(t *testing.T) { require.NoError(t, err) assert.Equal(t, test.expected, actual) if actual == nodeCustomDataPlatformUnsupported { - require.NoError(t, applyEmbeddedNodeCustomData(fstest.MapFS{}, releasePath), + outputPath := filepath.Join(t.TempDir(), "embedded-nodecustomdata.yml") + require.NoError(t, applyEmbeddedNodeCustomData(fstest.MapFS{}, releasePath, outputPath), "unsupported platforms must skip without reading any payload") + assert.NoFileExists(t, outputPath) } }) } @@ -47,8 +49,11 @@ func TestApplyEmbeddedNodeCustomData(t *testing.T) { for _, id := range []string{"ubuntu", "azurelinux"} { t.Run(id, func(t *testing.T) { directory := t.TempDir() - t.Setenv("TMPDIR", directory) - t.Setenv("TMP", directory) + outputPath := filepath.Join(directory, "containers", "embedded-nodecustomdata.yml") + legacyPath := filepath.Join(directory, "containers", "nodecustomdata.yml") + require.NoError(t, os.MkdirAll(filepath.Dir(legacyPath), 0o755)) + require.NoError(t, os.WriteFile(legacyPath, []byte("legacy payload"), 0o600)) + require.NoError(t, os.WriteFile(outputPath, []byte("previous embedded payload"), 0o600)) releasePath := filepath.Join(directory, "os-release") require.NoError(t, os.WriteFile(releasePath, []byte("ID="+id+"\n"), 0o600)) existing := filepath.Join(directory, "existing.sh") @@ -67,15 +72,22 @@ func TestApplyEmbeddedNodeCustomData(t *testing.T) { files := fstest.MapFS{ "scripthotfix/generated/rendered_nodecustomdata_" + platform + ".yml": &fstest.MapFile{Data: data}, } - require.NoError(t, applyEmbeddedNodeCustomData(files, releasePath)) + require.NoError(t, applyEmbeddedNodeCustomData(files, releasePath, outputPath)) for _, destination := range []string{existing, missing} { actual, readErr := os.ReadFile(destination) require.NoError(t, readErr) assert.Equal(t, payload, string(actual)) } - temporary, err := filepath.Glob(filepath.Join(directory, "aks-node-controller-nodecustomdata-*.yml")) + retained, err := os.ReadFile(outputPath) require.NoError(t, err) - assert.Empty(t, temporary) + assert.Equal(t, data, retained) + legacy, err := os.ReadFile(legacyPath) + require.NoError(t, err) + assert.Equal(t, "legacy payload", string(legacy)) + require.NoError(t, applyEmbeddedNodeCustomData(fstest.MapFS{}, releasePath, outputPath)) + retained, err = os.ReadFile(outputPath) + require.NoError(t, err) + assert.Equal(t, data, retained, "a skipped application must preserve the last payload") if runtime.GOOS != "windows" { info, statErr := os.Stat(missing) require.NoError(t, statErr) @@ -85,13 +97,14 @@ func TestApplyEmbeddedNodeCustomData(t *testing.T) { } } -func TestApplyEmbeddedNodeCustomDataErrorsAndCleanup(t *testing.T) { +func TestApplyEmbeddedNodeCustomDataErrorsAndRetention(t *testing.T) { tests := []struct { name string release string payload string missing string wantError string + retained bool }{ {name: "missing release", missing: "release", wantError: "read OS release"}, {name: "unknown OS", release: "ID=other", wantError: "unsupported OS ID"}, @@ -100,20 +113,22 @@ func TestApplyEmbeddedNodeCustomDataErrorsAndCleanup(t *testing.T) { {name: "missing payload", release: "ID=ubuntu", missing: "payload"}, {name: "other platform payload only", release: "ID=azurelinux", payload: "write_files: ["}, {name: "unreadable payload directory", release: "ID=ubuntu", missing: "payload-file", wantError: "read embedded nodecustomdata"}, - {name: "malformed YAML", release: "ID=ubuntu", payload: "write_files: [", wantError: "unmarshal nodecustomdata"}, - {name: "invalid entry", release: "ID=ubuntu", payload: "write_files:\n- content: invalid\n", wantError: "path is required"}, - {name: "empty payload", release: "ID=ubuntu", payload: "write_files: []\n"}, - {name: "temporary directory unavailable", release: "ID=ubuntu", payload: "write_files: []\n", missing: "temp", wantError: "create temporary nodecustomdata"}, + {name: "malformed YAML", release: "ID=ubuntu", payload: "write_files: [", wantError: "unmarshal nodecustomdata", retained: true}, + {name: "invalid entry", release: "ID=ubuntu", payload: "write_files:\n- content: invalid\n", wantError: "path is required", retained: true}, + {name: "empty payload", release: "ID=ubuntu", payload: "write_files: []\n", retained: true}, + {name: "output directory blocked", release: "ID=ubuntu", payload: "write_files: []\n", missing: "output-dir", wantError: "create embedded nodecustomdata directory"}, + {name: "output file blocked", release: "ID=ubuntu", payload: "write_files: []\n", missing: "output-file", wantError: "write embedded nodecustomdata"}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { directory := t.TempDir() - tempDir := directory - if test.missing == "temp" { - tempDir = filepath.Join(directory, "missing") + outputPath := filepath.Join(directory, "containers", "embedded-nodecustomdata.yml") + if test.missing == "output-dir" { + require.NoError(t, os.WriteFile(filepath.Dir(outputPath), []byte("blocked"), 0o600)) + } + if test.missing == "output-file" { + require.NoError(t, os.MkdirAll(outputPath, 0o755)) } - t.Setenv("TMPDIR", tempDir) - t.Setenv("TMP", tempDir) releasePath := filepath.Join(directory, "os-release") if test.missing != "release" { require.NoError(t, os.WriteFile(releasePath, []byte(test.release), 0o600)) @@ -127,15 +142,29 @@ func TestApplyEmbeddedNodeCustomDataErrorsAndCleanup(t *testing.T) { if test.missing == "payload-file" { files["scripthotfix/generated/rendered_nodecustomdata_ubuntu.yml"].Mode = fs.ModeDir } - err := applyEmbeddedNodeCustomData(files, releasePath) + err := applyEmbeddedNodeCustomData(files, releasePath, outputPath) if test.wantError == "" { require.NoError(t, err) } else { require.ErrorContains(t, err, test.wantError) } - temporary, err := filepath.Glob(filepath.Join(directory, "aks-node-controller-nodecustomdata-*.yml")) - require.NoError(t, err) - assert.Empty(t, temporary) + assertRetainedEmbeddedPayload(t, outputPath, test.payload, test.retained) }) } } + +func assertRetainedEmbeddedPayload(t *testing.T, outputPath, payload string, retained bool) { + t.Helper() + if !retained { + assert.NoFileExists(t, outputPath) + return + } + data, err := os.ReadFile(outputPath) + require.NoError(t, err) + assert.Equal(t, payload, string(data)) + if runtime.GOOS != "windows" { + info, err := os.Stat(outputPath) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o600), info.Mode().Perm()) + } +} From d7b378203ff38a2a022e2452d262f3fadc95b20f Mon Sep 17 00:00:00 2001 From: Devin Wong Date: Thu, 10 Sep 2026 17:29:43 -0700 Subject: [PATCH 26/26] Flatten embedded hotfix assets into generated directory Update embedding, generation, workflow tracking, fixtures, and documentation to use aks-node-controller/generated. Keep the on-node diagnostic YAML path unchanged. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4ae1dd11-ec06-4c26-bfc8-d1ef1f89fcf9 --- .github/workflows/hotfix-generate.yml | 2 +- aks-node-controller/README.md | 2 +- aks-node-controller/embeddednodecustomdata.go | 4 ++-- aks-node-controller/embeddednodecustomdata_test.go | 8 ++++---- aks-node-controller/{scripthotfix => }/generated/README | 0 e2e/scenario/vmss.go | 2 +- e2e/scenario/vmss_test.go | 7 +++---- hotfix/hotfix_generate.py | 2 +- 8 files changed, 13 insertions(+), 14 deletions(-) rename aks-node-controller/{scripthotfix => }/generated/README (100%) diff --git a/.github/workflows/hotfix-generate.yml b/.github/workflows/hotfix-generate.yml index 617d7e7aeac..50bd663540d 100644 --- a/.github/workflows/hotfix-generate.yml +++ b/.github/workflows/hotfix-generate.yml @@ -95,7 +95,7 @@ jobs: done < <( git status --porcelain --untracked-files=all -- \ parts/linux/cloud-init/artifacts/aks-node-controller-hotfix.json \ - aks-node-controller/scripthotfix/generated + aks-node-controller/generated ) if [ "${#FILES[@]}" -eq 0 ]; then echo "No template changes needed." diff --git a/aks-node-controller/README.md b/aks-node-controller/README.md index 151e40cb7a3..6cbe9b2fb19 100644 --- a/aks-node-controller/README.md +++ b/aks-node-controller/README.md @@ -165,7 +165,7 @@ is left untouched. Application errors are logged and provisioning continues. The embedded nodecustomdata coordinator distinguishes these script hotfixes from updates to the ANC binary itself. The generated files live under -`aks-node-controller/scripthotfix/generated/` as +`aks-node-controller/generated/` as `rendered_nodecustomdata_.yml`. The generator selects only changed hotfixable entries from `nodecustomdata.yml`, then renders only Ubuntu and standard Azure Linux variants through AgentBaker's production Go-template diff --git a/aks-node-controller/embeddednodecustomdata.go b/aks-node-controller/embeddednodecustomdata.go index fc33df49f62..4b74b361132 100644 --- a/aks-node-controller/embeddednodecustomdata.go +++ b/aks-node-controller/embeddednodecustomdata.go @@ -29,7 +29,7 @@ const ( nodeCustomDataPlatformUnsupported nodeCustomDataPlatform = "unsupported" ) -//go:embed scripthotfix/generated +//go:embed generated var embeddedGeneratedNodeCustomData embed.FS func applyEmbeddedNodeCustomData(payloadFS fs.FS, osReleasePath, outputPath string) error { @@ -44,7 +44,7 @@ func applyEmbeddedNodeCustomData(payloadFS fs.FS, osReleasePath, outputPath stri slog.Info("embedded script hotfix is not supported on this OS, skipping", "osReleasePath", osReleasePath) return nil } - renderedPath := fmt.Sprintf("scripthotfix/generated/rendered_nodecustomdata_%s.yml", platform) + renderedPath := fmt.Sprintf("generated/rendered_nodecustomdata_%s.yml", platform) data, err := fs.ReadFile(payloadFS, renderedPath) if errors.Is(err, fs.ErrNotExist) { return nil diff --git a/aks-node-controller/embeddednodecustomdata_test.go b/aks-node-controller/embeddednodecustomdata_test.go index 9a2001b35d7..c57aaae51a4 100644 --- a/aks-node-controller/embeddednodecustomdata_test.go +++ b/aks-node-controller/embeddednodecustomdata_test.go @@ -70,7 +70,7 @@ func TestApplyEmbeddedNodeCustomData(t *testing.T) { platform = string(nodeCustomDataPlatformMariner) } files := fstest.MapFS{ - "scripthotfix/generated/rendered_nodecustomdata_" + platform + ".yml": &fstest.MapFile{Data: data}, + "generated/rendered_nodecustomdata_" + platform + ".yml": &fstest.MapFile{Data: data}, } require.NoError(t, applyEmbeddedNodeCustomData(files, releasePath, outputPath)) for _, destination := range []string{existing, missing} { @@ -134,13 +134,13 @@ func TestApplyEmbeddedNodeCustomDataErrorsAndRetention(t *testing.T) { require.NoError(t, os.WriteFile(releasePath, []byte(test.release), 0o600)) } files := fstest.MapFS{ - "scripthotfix/generated/README": &fstest.MapFile{Data: []byte("placeholder")}, + "generated/README": &fstest.MapFile{Data: []byte("placeholder")}, } if test.missing != "payload" { - files["scripthotfix/generated/rendered_nodecustomdata_ubuntu.yml"] = &fstest.MapFile{Data: []byte(test.payload)} + files["generated/rendered_nodecustomdata_ubuntu.yml"] = &fstest.MapFile{Data: []byte(test.payload)} } if test.missing == "payload-file" { - files["scripthotfix/generated/rendered_nodecustomdata_ubuntu.yml"].Mode = fs.ModeDir + files["generated/rendered_nodecustomdata_ubuntu.yml"].Mode = fs.ModeDir } err := applyEmbeddedNodeCustomData(files, releasePath, outputPath) if test.wantError == "" { diff --git a/aks-node-controller/scripthotfix/generated/README b/aks-node-controller/generated/README similarity index 100% rename from aks-node-controller/scripthotfix/generated/README rename to aks-node-controller/generated/README diff --git a/e2e/scenario/vmss.go b/e2e/scenario/vmss.go index 58e538117e9..4190fc687b0 100644 --- a/e2e/scenario/vmss.go +++ b/e2e/scenario/vmss.go @@ -155,7 +155,7 @@ func writeScriptHotfixFixture(buildDir string, fixture ScriptHotfixFixture) erro return fmt.Errorf("script-hotfix fixture payload is empty") } - generatedDir := filepath.Join(buildDir, "scripthotfix", "generated") + generatedDir := filepath.Join(buildDir, "generated") rendered := scriptHotfixFixtureNodeCustomData{ WriteFiles: []scriptHotfixFixtureWriteFile{{ Path: fixture.Destination, diff --git a/e2e/scenario/vmss_test.go b/e2e/scenario/vmss_test.go index 98c4a6f3aa5..86e7e13ad07 100644 --- a/e2e/scenario/vmss_test.go +++ b/e2e/scenario/vmss_test.go @@ -15,7 +15,7 @@ import ( func TestWriteScriptHotfixFixture(t *testing.T) { buildDir := t.TempDir() - require.NoError(t, os.MkdirAll(filepath.Join(buildDir, "scripthotfix", "generated"), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(buildDir, "generated"), 0o755)) fixture := ScriptHotfixFixture{ Platform: "ubuntu", Destination: "/opt/azure/containers/provision_configs.sh", @@ -24,14 +24,13 @@ func TestWriteScriptHotfixFixture(t *testing.T) { } require.NoError(t, writeScriptHotfixFixture(buildDir, fixture)) - entries, err := os.ReadDir(filepath.Join(buildDir, "scripthotfix", "generated")) + entries, err := os.ReadDir(filepath.Join(buildDir, "generated")) require.NoError(t, err) require.Len(t, entries, 1) require.Equal(t, "rendered_nodecustomdata_ubuntu.yml", entries[0].Name()) renderedData, err := os.ReadFile(filepath.Join( buildDir, - "scripthotfix", "generated", "rendered_nodecustomdata_ubuntu.yml", )) @@ -88,7 +87,7 @@ func TestWriteScriptHotfixFixtureRejectsInvalidData(t *testing.T) { fixture := valid test.mutate(&fixture) buildDir := t.TempDir() - require.NoError(t, os.MkdirAll(filepath.Join(buildDir, "scripthotfix", "generated"), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(buildDir, "generated"), 0o755)) require.Error(t, writeScriptHotfixFixture(buildDir, fixture)) }) } diff --git a/hotfix/hotfix_generate.py b/hotfix/hotfix_generate.py index 9ec39f76ac9..caac26bffa3 100644 --- a/hotfix/hotfix_generate.py +++ b/hotfix/hotfix_generate.py @@ -42,7 +42,7 @@ ARTIFACTS_DIR = "parts/linux/cloud-init/artifacts" LINUX_SIG_VERSION_FILE = "pkg/agent/datamodel/linux_sig_version.json" ANC_DIR = "aks-node-controller/" -GENERATED_DIR = os.path.join(ANC_DIR, "scripthotfix", "generated") +GENERATED_DIR = os.path.join(ANC_DIR, "generated") VERSION_RE = re.compile(r'^\d{6}\.\d{2}\.\d+$')