feat: embed distro-specific rendered script hotfixes in ANC

.github/workflows/hotfix-generate.yml

@@ -2,9 +2,8 @@ name: Hotfix Template Update
 # Auto-detects whether a hotfix is needed for a PR targeting an official/* release
 # branch and, if so, computes the version numbers and updates the generated files:
 #   - If aks-node-controller/ changed vs the base branch, bumps `version`.
-#   - If parts/linux/cloud-init/nodecustomdata.yml ends up changed vs the base branch
-#     (either directly, or via auto-injection of changed CSE scripts), bumps
-#     `scripts_version`.
+#   - Changed CSE scripts are rendered into platform-specific nodecustomdata YAML
+#     files and embedded in the hotfix ANC package.
 #   - Writes the result to
 #     parts/linux/cloud-init/artifacts/aks-node-controller-hotfix.json (embedded
 #     directly into scriptless customData by pkg/agent/baker.go).
@@ -83,57 +82,80 @@ jobs:
 
       - name: Generate hotfix files
         run: |
+          python3 -m unittest hotfix.hotfix_generate_test
           python3 hotfix/hotfix_generate.py "origin/${GITHUB_BASE_REF}"
 
       - name: Commit changes via API
         env:
           GH_TOKEN: ${{ steps.app-token.outputs.token }}
-          HEAD_REF: ${{ github.head_ref }}
         run: |
-          FILES=(
-            "parts/linux/cloud-init/nodecustomdata.yml"
-            "parts/linux/cloud-init/artifacts/aks-node-controller-hotfix.json"
+          FILES=()
+          while IFS= read -r STATUS_LINE; do
+            FILES+=("${STATUS_LINE:3}")
+          done < <(
+            git status --porcelain --untracked-files=all -- \
+              parts/linux/cloud-init/artifacts/aks-node-controller-hotfix.json \
+              aks-node-controller/scripthotfix/generated
           )
-          CHANGED=0
+          if [ "${#FILES[@]}" -eq 0 ]; then
+            echo "No template changes needed."
+            exit 0
+          fi
+
+          REPOSITORY="repos/${{ github.repository }}"
+          BRANCH="${GITHUB_HEAD_REF}"
+          CHECKOUT_SHA=$(git rev-parse HEAD)
+          START_SHA=$(gh api "${REPOSITORY}/git/ref/heads/${BRANCH}" --jq '.object.sha')
+          if [ "$START_SHA" != "$CHECKOUT_SHA" ]; then
+            echo "Branch moved after checkout; refusing to commit stale generated content." >&2
+            exit 1
+          fi
+          BASE_TREE=$(gh api "${REPOSITORY}/git/commits/${START_SHA}" --jq '.tree.sha')
+          TREE='[]'
           for FILE in "${FILES[@]}"; do
-            # git diff --quiet misses untracked files (e.g. the target hotfix json on
-            # its first-ever generation on this branch), so use `git status
-            # --porcelain` instead, which reports both modified and untracked paths.
-            if [ -z "$(git status --porcelain -- "$FILE")" ]; then
-              continue
-            fi
-            CHANGED=1
-            # For a brand-new file the Contents API lookup 404s (no sha yet); the
-            # step runs under `bash -e`, so guard the lookup and only pass -f sha
-            # when the file already exists on the branch, otherwise the PUT must
-            # omit sha entirely to create the file.
-            SHA=$(gh api "repos/${{ github.repository }}/contents/${FILE}?ref=${HEAD_REF}" --jq '.sha' 2>/dev/null || true)
-            if [ ! -e "$FILE" ]; then
-              if [ -n "$SHA" ]; then
-                gh api "repos/${{ github.repository }}/contents/${FILE}" \
-                  -X DELETE \
-                  -f message="chore: auto-generate hotfix content for ${FILE}" \
-                  -f branch="${HEAD_REF}" \
-                  -f sha="$SHA"
-              fi
-              continue
-            fi
-            CONTENT=$(base64 -w 0 "$FILE")
-            if [ -n "$SHA" ]; then
-              gh api "repos/${{ github.repository }}/contents/${FILE}" \
-                -X PUT \
-                -f message="chore: auto-generate hotfix content for ${FILE}" \
-                -f content="$CONTENT" \
-                -f branch="${HEAD_REF}" \
-                -f sha="$SHA"
+            if [ -e "$FILE" ]; then
+              BLOB_SHA=$(
+                base64 -w 0 "$FILE" |
+                  jq -Rs '{content: ., encoding: "base64"}' |
+                  gh api "${REPOSITORY}/git/blobs" -X POST --input - --jq '.sha'
+              )
+              TREE=$(
+                jq -c \
+                  --arg path "$FILE" \
+                  --arg sha "$BLOB_SHA" \
+                  '. + [{path: $path, mode: "100644", type: "blob", sha: $sha}]' \
+                  <<< "$TREE"
+              )
             else
-              gh api "repos/${{ github.repository }}/contents/${FILE}" \
-                -X PUT \
-                -f message="chore: auto-generate hotfix content for ${FILE}" \
-                -f content="$CONTENT" \
-                -f branch="${HEAD_REF}"
+              TREE=$(
+                jq -c \
+                  --arg path "$FILE" \
+                  '. + [{path: $path, mode: "100644", type: "blob", sha: null}]' \
+                  <<< "$TREE"
+              )
             fi
           done
-          if [ "$CHANGED" -eq 0 ]; then
-            echo "No template changes needed."
+          NEW_TREE=$(
+            jq -n \
+              --arg base_tree "$BASE_TREE" \
+              --argjson tree "$TREE" \
+              '{base_tree: $base_tree, tree: $tree}' |
+              gh api "${REPOSITORY}/git/trees" -X POST --input - --jq '.sha'
+          )
+          NEW_COMMIT=$(
+            jq -n \
+              --arg message "chore: auto-generate hotfix content" \
+              --arg tree "$NEW_TREE" \
+              --arg parent "$START_SHA" \
+              '{message: $message, tree: $tree, parents: [$parent]}' |
+              gh api "${REPOSITORY}/git/commits" -X POST --input - --jq '.sha'
+          )
+          CURRENT_SHA=$(gh api "${REPOSITORY}/git/ref/heads/${BRANCH}" --jq '.object.sha')
+          if [ "$CURRENT_SHA" != "$START_SHA" ]; then
+            echo "Branch moved during generation; refusing to overwrite ${CURRENT_SHA}." >&2
+            exit 1
           fi
+          jq -n \
+            --arg sha "$NEW_COMMIT" \
+            '{sha: $sha, force: false}' |
+            gh api "${REPOSITORY}/git/refs/heads/${BRANCH}" -X PATCH --input -

aks-node-controller/README.md

@@ -149,3 +149,31 @@ Key components:
         ```
         This indicates the controller exited before emitting `provision.json`. Most commonly the rendered AKSNodeConfig was missing, had the wrong `Version` (expected `v1`), or was written to the wrong path (`/opt/azure/containers/aks-node-controller-config.json`). Fix the config generation, redeploy, and the bootstrap scripts will then populate `provision.json`.
 - **provision-wait**: waits for `provision.complete` to be present and reads `provision.json` which contains the provision output of type `CSEStatus` and is returned by CSE through capturing stdout.
+
+### Provisioning script hotfix payloads
+
+Patched ANC binaries can embed selected Linux provisioning scripts generated from
+`parts/linux/cloud-init/artifacts/`. At the start of `provision`, ANC validates the
+rendered nodecustomdata matching the local platform and atomically applies its
+`write_files` entries before constructing the normal CSE command.
+Application is fail-open so the existing VHD scripts remain usable if validation
+or replacement fails.
+
+The ANC-owned `scripthotfix` package distinguishes these embedded script hotfixes
+from updates to the ANC binary itself. The generated files live under
+`aks-node-controller/scripthotfix/generated/` as
+`rendered_nodecustomdata_<platform>.yml`. The generator selects only changed
+hotfixable entries from `nodecustomdata.yml`, then renders Ubuntu, Mariner/Azure
+Linux, ACL, OS Guard, and Flatcar variants through AgentBaker's production
+Go-template functions.
+
+Embedded payloads are replace-only: ANC skips an entry when its runtime
+destination does not already exist. File presence preserves non-platform
+template gates such as custom-image exclusions. New-file hotfixes are not
+supported by this delivery path.
+
+Script hotfix delivery is package-only. The existing base-to-version hotfix map
+selects the ANC package for the node's baked `YYYYMM.DD` version base; the package
+contains its corresponding rendered scripts. If the package cannot be installed,
+provisioning fails open to the original VHD scripts. The operational fallback is
+to upgrade the node image.

aks-node-controller/app.go

@@ -22,6 +22,7 @@ import (
 	"github.com/Azure/agentbaker/aks-node-controller/parser"
 	"github.com/Azure/agentbaker/aks-node-controller/pkg/gpu"
 	"github.com/Azure/agentbaker/aks-node-controller/pkg/nodeconfigutils"
+	"github.com/Azure/agentbaker/aks-node-controller/scripthotfix"
 	"github.com/fsnotify/fsnotify"
 	"github.com/urfave/cli/v3"
 )
@@ -71,6 +72,8 @@ type App struct {
 	// Authorization header for the check-hotfix LPS fetch. When nil, the real IMDS endpoint
 	// is queried.
 	fetchAttestedToken func(ctx context.Context) (string, error)
+	// applyEmbeddedHotfix overrides embedded script application for tests.
+	applyEmbeddedHotfix func(string) (scripthotfix.Result, error)
 	// grpcDialContext overrides how the gRPC LPS client dials, letting tests point the client at
 	// an in-process (bufconn) server. When nil, the real TLS dial to the apiserver front is used.
 	grpcDialContext func(ctx context.Context, target string) (net.Conn, error)
@@ -684,6 +687,23 @@ func (a *App) Provision(ctx context.Context, flags ProvisionFlags) (*ProvisionRe
 	return provisionResult, err
 }
 
+func (a *App) applyEmbeddedHotfixPayload() {
+	applyEmbeddedHotfix := a.applyEmbeddedHotfix
+	if applyEmbeddedHotfix == nil {
+		applyEmbeddedHotfix = scripthotfix.ApplyEmbedded
+	}
+	result, err := applyEmbeddedHotfix(a.osReleasePath)
+	if err != nil {
+		// Hotfixes are fail-open: the VHD-baked scripts remain available, so an
+		// embedded payload must not block provisioning.
+		slog.Warn("failed to apply embedded hotfix payload; continuing with existing scripts",
+			"error", err)
+	} else if result.Applied > 0 || result.Skipped > 0 {
+		slog.Info("processed embedded hotfix payload",
+			"applied", result.Applied, "skipped", result.Skipped)
+	}
+}
+
 // runProvision encapsulates execution for the "provision" subcommand after CLI parsing.
 // It returns an error describing any failure; callers should pass that error to
 // writeCompleteFileOnError so the sentinel file can be written on fail-fast paths.
@@ -709,6 +729,8 @@ func (a *App) runProvision(ctx context.Context, flags ProvisionFlags, dryRun boo
 	}
 	if dryRun {
 		a.cmdRun = cmdRunnerDryRun
+	} else {
+		a.applyEmbeddedHotfixPayload()
 	}
 	return a.Provision(ctx, flags)
 }

aks-node-controller/app_test.go

@@ -15,6 +15,7 @@ import (
 	"time"
 
 	"github.com/Azure/agentbaker/aks-node-controller/helpers"
+	"github.com/Azure/agentbaker/aks-node-controller/scripthotfix"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -231,6 +232,70 @@ func TestApp_Run(t *testing.T) {
 }
 
 func TestApp_Provision(t *testing.T) {
+	t.Run("embedded hotfix runs before command construction and execution", func(t *testing.T) {
+		tt := NewTestApp(t, TestAppConfig{})
+		applied := false
+		tt.App.applyEmbeddedHotfix = func(string) (scripthotfix.Result, error) {
+			applied = true
+			return scripthotfix.Result{Applied: 1}, nil
+		}
+
+		_, err := tt.App.runProvision(
+			context.Background(),
+			ProvisionFlags{ProvisionConfig: "does-not-exist.json"},
+			false,
+		)
+
+		require.Error(t, err)
+		assert.True(t, applied, "embedded payload must run before config parsing")
+	})
+
+	t.Run("embedded hotfix failure is logged and provisioning continues", func(t *testing.T) {
+		logs := installLogCapturer(t)
+		executed := false
+		tt := NewTestApp(t, TestAppConfig{
+			RunFunc: func(*exec.Cmd) error {
+				executed = true
+				return nil
+			},
+		})
+		tt.App.applyEmbeddedHotfix = func(string) (scripthotfix.Result, error) {
+			return scripthotfix.Result{}, errors.New("rendered nodecustomdata validation failed")
+		}
+
+		_, err := tt.App.runProvision(
+			context.Background(),
+			ProvisionFlags{NBCCmd: "parser/testdata/test_nbccmd.sh"},
+			false,
+		)
+
+		require.NoError(t, err)
+		assert.True(t, executed)
+		assert.Contains(t, logs.getRecords(), logRecord{
+			Level:   slog.LevelWarn,
+			Message: "failed to apply embedded hotfix payload; continuing with existing scripts",
+			Attrs:   map[string]string{"error": "rendered nodecustomdata validation failed"},
+		})
+	})
+
+	t.Run("dry-run does not apply embedded hotfix payload", func(t *testing.T) {
+		tt := NewTestApp(t, TestAppConfig{})
+		applied := false
+		tt.App.applyEmbeddedHotfix = func(string) (scripthotfix.Result, error) {
+			applied = true
+			return scripthotfix.Result{}, nil
+		}
+
+		_, err := tt.App.runProvision(
+			context.Background(),
+			ProvisionFlags{NBCCmd: "parser/testdata/test_nbccmd.sh"},
+			true,
+		)
+
+		require.NoError(t, err)
+		assert.False(t, applied)
+	})
+
 	t.Run("valid provision config", func(t *testing.T) {
 		tt := NewTestApp(t, TestAppConfig{})
 		_, err := tt.App.Provision(context.Background(), ProvisionFlags{ProvisionConfig: "parser/testdata/test_aksnodeconfig.json"})