feat: embed distro-specific rendered script hotfixes in ANC

.github/workflows/hotfix-generate.yml

@@ -2,9 +2,8 @@ name: Hotfix Template Update
 # Auto-detects whether a hotfix is needed for a PR targeting an official/* release
 # branch and, if so, computes the version numbers and updates the generated files:
 #   - If aks-node-controller/ changed vs the base branch, bumps `version`.
-#   - If parts/linux/cloud-init/nodecustomdata.yml ends up changed vs the base branch
-#     (either directly, or via auto-injection of changed CSE scripts), bumps
-#     `scripts_version`.
+#   - Changed CSE scripts are rendered into platform-specific nodecustomdata YAML
+#     files and embedded in the hotfix ANC package.
 #   - Writes the result to
 #     parts/linux/cloud-init/artifacts/aks-node-controller-hotfix.json (embedded
 #     directly into scriptless customData by pkg/agent/baker.go).
@@ -83,57 +82,80 @@ jobs:
 
       - name: Generate hotfix files
         run: |
+          python3 -m unittest hotfix.hotfix_generate_test
           python3 hotfix/hotfix_generate.py "origin/${GITHUB_BASE_REF}"
 
       - name: Commit changes via API
         env:
           GH_TOKEN: ${{ steps.app-token.outputs.token }}
-          HEAD_REF: ${{ github.head_ref }}
         run: |
-          FILES=(
-            "parts/linux/cloud-init/nodecustomdata.yml"
-            "parts/linux/cloud-init/artifacts/aks-node-controller-hotfix.json"
+          FILES=()
+          while IFS= read -r STATUS_LINE; do
+            FILES+=("${STATUS_LINE:3}")
+          done < <(
+            git status --porcelain --untracked-files=all -- \
+              parts/linux/cloud-init/artifacts/aks-node-controller-hotfix.json \
+              aks-node-controller/scripthotfix/generated
           )
-          CHANGED=0
+          if [ "${#FILES[@]}" -eq 0 ]; then
+            echo "No template changes needed."
+            exit 0
+          fi
+
+          REPOSITORY="repos/${{ github.repository }}"
+          BRANCH="${GITHUB_HEAD_REF}"
+          CHECKOUT_SHA=$(git rev-parse HEAD)
+          START_SHA=$(gh api "${REPOSITORY}/git/ref/heads/${BRANCH}" --jq '.object.sha')
+          if [ "$START_SHA" != "$CHECKOUT_SHA" ]; then
+            echo "Branch moved after checkout; refusing to commit stale generated content." >&2
+            exit 1
+          fi
+          BASE_TREE=$(gh api "${REPOSITORY}/git/commits/${START_SHA}" --jq '.tree.sha')
+          TREE='[]'
           for FILE in "${FILES[@]}"; do
-            # git diff --quiet misses untracked files (e.g. the target hotfix json on
-            # its first-ever generation on this branch), so use `git status
-            # --porcelain` instead, which reports both modified and untracked paths.
-            if [ -z "$(git status --porcelain -- "$FILE")" ]; then
-              continue
-            fi
-            CHANGED=1
-            # For a brand-new file the Contents API lookup 404s (no sha yet); the
-            # step runs under `bash -e`, so guard the lookup and only pass -f sha
-            # when the file already exists on the branch, otherwise the PUT must
-            # omit sha entirely to create the file.
-            SHA=$(gh api "repos/${{ github.repository }}/contents/${FILE}?ref=${HEAD_REF}" --jq '.sha' 2>/dev/null || true)
-            if [ ! -e "$FILE" ]; then
-              if [ -n "$SHA" ]; then
-                gh api "repos/${{ github.repository }}/contents/${FILE}" \
-                  -X DELETE \
-                  -f message="chore: auto-generate hotfix content for ${FILE}" \
-                  -f branch="${HEAD_REF}" \
-                  -f sha="$SHA"
-              fi
-              continue
-            fi
-            CONTENT=$(base64 -w 0 "$FILE")
-            if [ -n "$SHA" ]; then
-              gh api "repos/${{ github.repository }}/contents/${FILE}" \
-                -X PUT \
-                -f message="chore: auto-generate hotfix content for ${FILE}" \
-                -f content="$CONTENT" \
-                -f branch="${HEAD_REF}" \
-                -f sha="$SHA"
+            if [ -e "$FILE" ]; then
+              BLOB_SHA=$(
+                base64 -w 0 "$FILE" |
+                  jq -Rs '{content: ., encoding: "base64"}' |
+                  gh api "${REPOSITORY}/git/blobs" -X POST --input - --jq '.sha'
+              )
+              TREE=$(
+                jq -c \
+                  --arg path "$FILE" \
+                  --arg sha "$BLOB_SHA" \
+                  '. + [{path: $path, mode: "100644", type: "blob", sha: $sha}]' \
+                  <<< "$TREE"
+              )
             else
-              gh api "repos/${{ github.repository }}/contents/${FILE}" \
-                -X PUT \
-                -f message="chore: auto-generate hotfix content for ${FILE}" \
-                -f content="$CONTENT" \
-                -f branch="${HEAD_REF}"
+              TREE=$(
+                jq -c \
+                  --arg path "$FILE" \
+                  '. + [{path: $path, mode: "100644", type: "blob", sha: null}]' \
+                  <<< "$TREE"
+              )
             fi
           done
-          if [ "$CHANGED" -eq 0 ]; then
-            echo "No template changes needed."
+          NEW_TREE=$(
+            jq -n \
+              --arg base_tree "$BASE_TREE" \
+              --argjson tree "$TREE" \
+              '{base_tree: $base_tree, tree: $tree}' |
+              gh api "${REPOSITORY}/git/trees" -X POST --input - --jq '.sha'
+          )
+          NEW_COMMIT=$(
+            jq -n \
+              --arg message "chore: auto-generate hotfix content" \
+              --arg tree "$NEW_TREE" \
+              --arg parent "$START_SHA" \
+              '{message: $message, tree: $tree, parents: [$parent]}' |
+              gh api "${REPOSITORY}/git/commits" -X POST --input - --jq '.sha'
+          )
+          CURRENT_SHA=$(gh api "${REPOSITORY}/git/ref/heads/${BRANCH}" --jq '.object.sha')
+          if [ "$CURRENT_SHA" != "$START_SHA" ]; then
+            echo "Branch moved during generation; refusing to overwrite ${CURRENT_SHA}." >&2
+            exit 1
           fi
+          jq -n \
+            --arg sha "$NEW_COMMIT" \
+            '{sha: $sha, force: false}' |
+            gh api "${REPOSITORY}/git/refs/heads/${BRANCH}" -X PATCH --input -

aks-node-controller/README.md

@@ -149,3 +149,53 @@ Key components:
         ```
         This indicates the controller exited before emitting `provision.json`. Most commonly the rendered AKSNodeConfig was missing, had the wrong `Version` (expected `v1`), or was written to the wrong path (`/opt/azure/containers/aks-node-controller-config.json`). Fix the config generation, redeploy, and the bootstrap scripts will then populate `provision.json`.
 - **provision-wait**: waits for `provision.complete` to be present and reads `provision.json` which contains the provision output of type `CSEStatus` and is returned by CSE through capturing stdout.
+
+### Provisioning script hotfix payloads
+
+Patched ANC binaries can embed selected Linux provisioning scripts generated from
+`parts/linux/cloud-init/artifacts/`. At the start of `provision`, ANC writes the
+rendered nodecustomdata matching the local platform to
+`/opt/azure/containers/embedded-nodecustomdata.yml` (mode `0600` on creation)
+and calls the existing `applyNodeCustomData` function before constructing the
+normal CSE command. This file is retained for debugging, separate from the legacy
+`nodecustomdata.yml`. It contains the most recently written payload, including
+when application fails; the `applied embedded hotfix payload` log confirms
+successful application. If no payload is selected, any previously retained file
+is left untouched. Application errors are logged and provisioning continues.
+
+The embedded nodecustomdata coordinator distinguishes these script hotfixes from
+updates to the ANC binary itself. The generated files live under
+`aks-node-controller/scripthotfix/generated/` as
+`rendered_nodecustomdata_<platform>.yml`. The generator selects only changed
+hotfixable entries from `nodecustomdata.yml`, then renders only Ubuntu and
+standard Azure Linux variants through AgentBaker's production Go-template
+functions. Azure Linux retains the `mariner` payload filename; the legacy
+`ID=mariner` OS is no longer supported. OS Guard, ACL, and Flatcar are explicitly skipped during embedded
+application, including variants that share the `azurelinux` OS ID. Their
+distro-specific source changes do not trigger payload generation.
+
+The repository keeps a README placeholder so `go:embed` builds without any
+script hotfix payloads. Generation replaces it with Ubuntu and Azure Linux YAMLs.
+ANC skips application when the local platform's YAML is absent; no separate
+activation flag is needed. Other payload read errors are logged.
+
+When a PR has no new script hotfix, generation leaves the existing rendered
+payload unchanged. The active ANC version pointer is likewise retained until it
+is retired explicitly.
+
+The existing applier writes entries sequentially and creates missing destination
+files and parent directories. There is no transactional rollback: if an entry
+fails, earlier writes remain. Generation selects by distro only; hotfix authors
+must separately account for non-distro template conditions such as custom-image
+exclusions.
+
+`cse_start.sh` (`provision_start.sh` on the node) is excluded from embedded
+hotfixes to preserve custom-image wrappers. Generation fails explicitly if this
+script differs from the VHD baseline, even when other scripts also changed.
+Wrapper fixes require a new node image until runtime eligibility is available.
+
+Script hotfix delivery is package-only. The existing base-to-version hotfix map
+selects the ANC package for the node's baked `YYYYMM.DD` version base; the package
+contains its corresponding rendered scripts. If the package cannot be installed,
+provisioning fails open to the original VHD scripts. The operational fallback is
+to upgrade the node image.

aks-node-controller/app.go

@@ -72,6 +72,8 @@ type App struct {
 	// Authorization header for the check-hotfix LPS fetch. When nil, the real IMDS endpoint
 	// is queried.
 	fetchAttestedToken func(ctx context.Context) (string, error)
+	// applyEmbeddedHotfix overrides embedded script application for tests.
+	applyEmbeddedHotfix func(string) error
 	// grpcDialContext overrides how the gRPC LPS client dials, letting tests point the client at
 	// an in-process (bufconn) server. When nil, the real TLS dial to the apiserver front is used.
 	grpcDialContext func(ctx context.Context, target string) (net.Conn, error)
@@ -703,6 +705,16 @@ func (a *App) runProvision(ctx context.Context, flags ProvisionFlags, dryRun boo
 	}
 	if dryRun {
 		a.cmdRun = cmdRunnerDryRun
+	} else {
+		applyHotfix := a.applyEmbeddedHotfix
+		if applyHotfix == nil {
+			applyHotfix = func(osReleasePath string) error {
+				return applyEmbeddedNodeCustomData(embeddedGeneratedNodeCustomData, osReleasePath, embeddedNodeCustomDataPath)
+			}
+		}
+		if err := applyHotfix(a.osReleasePath); err != nil {
+			slog.Warn("failed to apply embedded hotfix payload; continuing provisioning", "error", err)
+		}
 	}
 	return a.Provision(ctx, flags)
 }

aks-node-controller/app_test.go

@@ -231,6 +231,70 @@ func TestApp_Run(t *testing.T) {
 }
 
 func TestApp_Provision(t *testing.T) {
+	t.Run("embedded hotfix runs before command construction and execution", func(t *testing.T) {
+		tt := NewTestApp(t, TestAppConfig{})
+		applied := false
+		tt.App.applyEmbeddedHotfix = func(string) error {
+			applied = true
+			return nil
+		}
+
+		_, err := tt.App.runProvision(
+			context.Background(),
+			ProvisionFlags{ProvisionConfig: "does-not-exist.json"},
+			false,
+		)
+
+		require.Error(t, err)
+		assert.True(t, applied, "embedded payload must run before config parsing")
+	})
+
+	t.Run("embedded hotfix failure is logged and provisioning continues", func(t *testing.T) {
+		logs := installLogCapturer(t)
+		executed := false
+		tt := NewTestApp(t, TestAppConfig{
+			RunFunc: func(*exec.Cmd) error {
+				executed = true
+				return nil
+			},
+		})
+		tt.App.applyEmbeddedHotfix = func(string) error {
+			return errors.New("rendered nodecustomdata application failed")
+		}
+
+		_, err := tt.App.runProvision(
+			context.Background(),
+			ProvisionFlags{NBCCmd: "parser/testdata/test_nbccmd.sh"},
+			false,
+		)
+
+		require.NoError(t, err)
+		assert.True(t, executed)
+		assert.Contains(t, logs.getRecords(), logRecord{
+			Level:   slog.LevelWarn,
+			Message: "failed to apply embedded hotfix payload; continuing provisioning",
+			Attrs:   map[string]string{"error": "rendered nodecustomdata application failed"},
+		})
+	})
+
+	t.Run("dry-run does not apply embedded hotfix payload", func(t *testing.T) {
+		tt := NewTestApp(t, TestAppConfig{})
+		applied := false
+		tt.App.applyEmbeddedHotfix = func(string) error {
+			applied = true
+			return nil
+		}
+
+		_, err := tt.App.runProvision(
+			context.Background(),
+			ProvisionFlags{NBCCmd: "parser/testdata/test_nbccmd.sh"},
+			true,
+		)
+
+		require.NoError(t, err)
+		assert.False(t, applied)
+	})
+
 	t.Run("valid provision config", func(t *testing.T) {
 		tt := NewTestApp(t, TestAppConfig{})
 		_, err := tt.App.Provision(context.Background(), ProvisionFlags{ProvisionConfig: "parser/testdata/test_aksnodeconfig.json"})

aks-node-controller/checkhotfix.go

@@ -103,6 +103,13 @@ const (
 	outcomeNoHotfixAvailable checkHotfixOutcome = "noHotfixAvailable"
 	// outcomeCustomDataFallback: LPS read failed; the embedded customdata pointer was used.
 	outcomeCustomDataFallback checkHotfixOutcome = "customDataFallback"
+	// outcomeNoColdStartPointer: the LPS could not be reached and the node config carried no
+	// cold-start hotfixes map, so there was nothing to stage. This is benign and expected on a
+	// node whose config was seeded without an injected map: download-hotfix simply keeps the
+	// existing on-disk pointer (the single-version one cloud-init wrote). Nothing failed, so it
+	// must not be reported at error level; the wrapped fetch error is still carried in the
+	// telemetry message to preserve why the LPS was unreachable.
+	outcomeNoColdStartPointer checkHotfixOutcome = "noColdStartPointer"
 	// outcomeFailed: everything failed; nothing was staged. Provisioning still proceeds (exit 0).
 	outcomeFailed checkHotfixOutcome = "failed"
 )
@@ -164,8 +171,13 @@ func (a *App) runCheckHotfixCommand(ctx context.Context) (err error) {
 	message := fmt.Sprintf("check-hotfix outcome=%s", outcome)
 	if err != nil {
 		message = fmt.Sprintf("%s error=%s", message, err.Error())
+	}
+	switch {
+	case level == helpers.EventLevelError:
 		slog.Warn("check-hotfix completed with error (fail-open)", "outcome", outcome, "error", err)
-	} else {
+	case err != nil:
+		slog.Info("check-hotfix completed (fail-open)", "outcome", outcome, "reason", err)
+	default:
 		slog.Info("check-hotfix completed", "outcome", outcome)
 	}
 	if a.eventLogger != nil {
@@ -253,7 +265,10 @@ func (a *App) handleFetchError(hotfixPath string, fetchErr error) (checkHotfixOu
 		return outcomeFailed, fmt.Errorf("LPS fetch failed (%w) and cold-start fallback failed: %w", fetchErr, coldErr)
 	}
 	if !ok {
-		return outcomeFailed, fmt.Errorf("LPS fetch failed and no cold-start pointer present: %w", fetchErr)
+		// Benign: no map was injected into the node config, so there is nothing to stage and
+		// the existing on-disk pointer stays intact. Report a non-error outcome while keeping
+		// the fetch error for diagnosis of why the LPS was unreachable.
+		return outcomeNoColdStartPointer, fmt.Errorf("LPS fetch failed and no cold-start pointer present: %w", fetchErr)
 	}
 	if err := writeHotfixConfig(hotfixPath, cfg); err != nil {
 		return outcomeFailed, fmt.Errorf("writing cold-start hotfix config: %w", err)