Adopt authoritative AKS Machine goals

docs/design.md

@@ -152,7 +152,7 @@ See [AKS RP And Flex Node Agent Interaction](design/agent-and-aks.md) for the de
 
 ## State And Idempotency
 
-The agent persists local daemon state so it can recover after restart, reboot, or partial failure. Persisted state includes the applied Kubernetes/settings version and active nspawn machine side.
+The agent persists local daemon state so it can recover after restart, reboot, or partial failure. Persisted state includes the current and previous applied Machine goals and the active nspawn machine side.
 
 The current state model separates desired state, applied state, and runtime discovery:
 

docs/design/in-cluster-machine.md

@@ -15,7 +15,7 @@ The controller serves the `armcontainerservice.Machine` JSON shape from the `kub
       "orchestratorVersion": "1.34.0",
       "maxPods": 110,
       "nodeLabels": {
-        "kubernetes.azure.com/managed": "false"
+        "workload": "edge"
       }
     },
     "provisioningState": "Succeeded"
@@ -27,18 +27,18 @@ Status updates use a separate patch model because the agent operation status is
 
 ## Bootstrap flow
 
-The local bootstrap configuration is authoritative while `aks-flex-node start` is running:
+The local bootstrap configuration seeds a Machine when one does not already exist. Once the endpoint returns a Machine, its complete goal is authoritative for bootstrap:
 
 1. `NewMachineClient` selects the in-cluster backend without a supplied Kubernetes REST config.
 2. The client builds a REST config from the bootstrap token or configured exec credential.
 3. `EnsureMachine` reads the machine through the Kubernetes service proxy.
 4. If the machine is absent, the client sends a PUT using the local bootstrap goal.
-5. If its Kubernetes version differs, the client sends a PUT that overwrites the remote goal with the local version.
-6. If its Kubernetes version already matches, local bootstrap settings remain authoritative; remote settings other than the ETag do not replace them.
-7. The returned ETag becomes the reconciliation baseline for the locally applied goal.
-8. The daemon state is seeded from that ETag before host or nspawn state is mutated. A later ETag change is treated as a new remote goal.
+5. Whether read or created, the returned Machine is validated and its goal replaces the local bootstrap goal. This includes Kubernetes version, max pods, custom labels, taints, kubelet image-GC thresholds, and the ETag-backed settings version. Scalar defaults omitted by the API retain their validated local bootstrap values.
+6. The daemon resolves nspawn settings and seeds its state from that same effective goal before mutating the host. A later ETag change is treated as a new remote goal.
 
-The ConfigMap-backed controller is read-only: it accepts mutation requests but returns the pre-created machine. Its fixture must therefore already match the local bootstrap version. When machine registration is required, a mismatch fails bootstrap before host mutation.
+When `orchestratorVersion` is a `major.minor` alias, the returned `currentOrchestratorVersion` supplies the exact patch used for artifact resolution.
+
+The ConfigMap-backed controller is read-only: it accepts mutation requests but returns the pre-created Machine. The agent adopts that returned goal even when it differs from local bootstrap configuration. When registration is required, a read, create, or validation failure stops bootstrap before host mutation. When registration is optional, bootstrap continues with the local goal.
 
 ## Daemon flow
 

hack/demo/aks-flex-node-upgrade.sh

@@ -131,7 +131,7 @@ update_machine_goal() {
       .properties.eTag = $settings |
       .properties.kubernetes = (.properties.kubernetes // {}) |
       .properties.kubernetes.orchestratorVersion = $version |
-      .properties.kubernetes.nodeLabels = (.properties.kubernetes.nodeLabels // {"kubernetes.azure.com/managed":"false"})
+      .properties.kubernetes.nodeLabels = (.properties.kubernetes.nodeLabels // {})
     ' <<<"${current_json}" > "${tmp}"
 
   if [[ -z "${cm_json}" ]]; then

hack/e2e/lib/controller.sh

@@ -332,7 +332,7 @@ ensure_flex_controller() {
 }
 
 _render_machine_json() {
-  local node_name="$1" kubernetes_version="$2" settings_version="$3"
+  local node_name="$1" kubernetes_version="$2" settings_version="$3" max_pods="$4"
   local cluster_id machine_id
   cluster_id="$(state_get cluster_id)"
   machine_id="${cluster_id}/agentPools/${E2E_TARGET_AGENT_POOL_NAME}/machines/${node_name}"
@@ -342,6 +342,7 @@ _render_machine_json() {
     --arg name "${node_name}" \
     --arg kubernetesVersion "${kubernetes_version}" \
     --arg eTag "${settings_version}" \
+    --argjson maxPods "${max_pods}" \
     '{
       id: $id,
       name: $name,
@@ -351,7 +352,7 @@ _render_machine_json() {
         provisioningState: "Succeeded",
         kubernetes: {
           orchestratorVersion: $kubernetesVersion,
-          maxPods: 110,
+          maxPods: $maxPods,
           nodeLabels: {},
           nodeTaints: [],
           kubeletConfig: {
@@ -364,11 +365,11 @@ _render_machine_json() {
 }
 
 _machine_configmap_upsert_unlocked() {
-  local node_name="$1" kubernetes_version="$2" settings_version="$3"
+  local node_name="$1" kubernetes_version="$2" settings_version="$3" max_pods="$4"
   local machine_file patch
   machine_file="${E2E_WORK_DIR}/machine-${node_name}.json"
 
-  _render_machine_json "${node_name}" "${kubernetes_version}" "${settings_version}" > "${machine_file}"
+  _render_machine_json "${node_name}" "${kubernetes_version}" "${settings_version}" "${max_pods}" > "${machine_file}"
   if ! kubectl -n "${E2E_CONTROLLER_NAMESPACE}" get configmap "${E2E_MACHINE_CONFIGMAP}" >/dev/null 2>&1; then
     kubectl -n "${E2E_CONTROLLER_NAMESPACE}" create configmap "${E2E_MACHINE_CONFIGMAP}" >/dev/null
   fi
@@ -379,8 +380,8 @@ _machine_configmap_upsert_unlocked() {
 }
 
 machine_configmap_upsert() {
-  local node_name="$1" kubernetes_version="${2:-${E2E_KUBERNETES_VERSION}}" settings_version="${3:-${kubernetes_version}}"
-  with_cluster_lock _machine_configmap_upsert_unlocked "${node_name}" "${kubernetes_version}" "${settings_version}"
+  local node_name="$1" kubernetes_version="${2:-${E2E_KUBERNETES_VERSION}}" settings_version="${3:-${kubernetes_version}}" max_pods="${4:-110}"
+  with_cluster_lock _machine_configmap_upsert_unlocked "${node_name}" "${kubernetes_version}" "${settings_version}" "${max_pods}"
 }
 
 _machine_configmap_delete_unlocked() {

hack/e2e/lib/node-join-token.sh

@@ -95,7 +95,7 @@ node_join_token() {
   mv "${config_file}.tmp" "${config_file}"
 
   # Step 3: Publish the AKS Machine goal and deploy the agent.
-  machine_configmap_upsert "$(state_get token_vm_name)" "${E2E_KUBERNETES_VERSION}" "${E2E_KUBERNETES_VERSION}"
+  machine_configmap_upsert "$(state_get token_vm_name)" "${E2E_KUBERNETES_VERSION}" "${E2E_KUBERNETES_VERSION}" "${E2E_KUBELET_MAX_PODS}"
   _deploy_and_start_agent "${vm_ip}" "${config_file}" "aks-flex-node-token"
 
   log_success "Token node joined in $(timer_elapsed "${start}")s"

hack/e2e/lib/upgrade-drift.sh

@@ -102,12 +102,16 @@ _ensure_mode_joined() {
 
 _trigger_mode_repave() {
   local mode="$1" desired_version="$2" settings_version="$3"
-  local vm_ip vm_name
+  local vm_ip vm_name max_pods
   vm_ip="$(_mode_vm_ip "${mode}")"
   vm_name="$(_mode_vm_name "${mode}")"
+  max_pods="110"
+  if [[ "${mode}" == "token" ]]; then
+    max_pods="${E2E_KUBELET_MAX_PODS}"
+  fi
 
   log_info "Updating controller machine goal for ${mode} node to Kubernetes ${desired_version} (${settings_version})"
-  machine_configmap_upsert "${vm_name}" "${desired_version}" "${settings_version}"
+  machine_configmap_upsert "${vm_name}" "${desired_version}" "${settings_version}" "${max_pods}"
   remote_exec "${vm_ip}" 'sudo systemctl status aks-flex-node-agent.service --no-pager -l || true'
 
   log_info "Deleting Kubernetes Node ${vm_name} to trigger ${mode} repave"

pkg/aksmachine/client_armapi.go

@@ -47,7 +47,7 @@ func newARMClient(cfg *config.Config, logger *slog.Logger) (MachineClient, error
 }
 
 func (c *armMachineClient) Create(ctx context.Context, desired GoalState) (*Machine, error) {
-	if err := desired.validate(); err != nil {
+	if err := desired.Validate(); err != nil {
 		return nil, fmt.Errorf("validate goal state: %w", err)
 	}
 	params := armcontainerservice.Machine{
@@ -275,6 +275,12 @@ func machineFromARM(machine armcontainerservice.Machine, defaultID, defaultName
 		kubernetes := properties.Kubernetes
 		if kubernetes.OrchestratorVersion != nil {
 			result.Goal.KubernetesVersion = *kubernetes.OrchestratorVersion
+			if kubernetes.CurrentOrchestratorVersion != nil {
+				result.Goal.KubernetesVersion = resolveKubernetesVersionAlias(
+					result.Goal.KubernetesVersion,
+					*kubernetes.CurrentOrchestratorVersion,
+				)
+			}
 		}
 		if kubernetes.MaxPods != nil {
 			result.Goal.MaxPods = int(*kubernetes.MaxPods)
@@ -290,7 +296,8 @@ func machineFromARM(machine armcontainerservice.Machine, defaultID, defaultName
 				result.Goal.KubeletConfig.ImageGCHighThreshold = int(*kubernetes.KubeletConfig.ImageGcHighThreshold)
 			}
 			if kubernetes.KubeletConfig.ImageGcLowThreshold != nil {
-				result.Goal.KubeletConfig.ImageGCLowThreshold = int(*kubernetes.KubeletConfig.ImageGcLowThreshold)
+				lowThreshold := int(*kubernetes.KubeletConfig.ImageGcLowThreshold)
+				result.Goal.KubeletConfig.ImageGCLowThreshold = &lowThreshold
 			}
 		}
 	}
@@ -303,6 +310,15 @@ func machineFromARM(machine armcontainerservice.Machine, defaultID, defaultName
 	return result
 }
 
+func resolveKubernetesVersionAlias(desired, current string) string {
+	desiredVersion := strings.TrimPrefix(strings.TrimSpace(desired), "v")
+	currentVersion := strings.TrimPrefix(strings.TrimSpace(current), "v")
+	if len(strings.Split(desiredVersion, ".")) == 2 && strings.HasPrefix(currentVersion, desiredVersion+".") {
+		return currentVersion
+	}
+	return desiredVersion
+}
+
 func stringMapFromPointers(values map[string]*string) map[string]string {
 	result := make(map[string]string, len(values))
 	for key, value := range values {

pkg/aksmachine/client_armapi_test.go

@@ -240,39 +240,62 @@ func TestGoalStateValidate(t *testing.T) {
 	}{
 		{
 			name: "valid",
-			goal: GoalState{KubernetesVersion: "1.35.1"},
+			goal: testGoal("1.35.1", ""),
 		},
 		{
 			name:    "missing Kubernetes version",
 			goal:    GoalState{},
 			wantErr: "kubernetes version is empty",
 		},
 		{
-			name:    "negative max pods",
-			goal:    GoalState{KubernetesVersion: "1.35.1", MaxPods: -1},
-			wantErr: "max pods must be non-negative",
+			name:    "missing max pods",
+			goal:    GoalState{KubernetesVersion: "1.35.1"},
+			wantErr: "max pods must be positive",
 		},
 		{
-			name:    "max pods exceeds int32",
-			goal:    GoalState{KubernetesVersion: "1.35.1", MaxPods: math.MaxInt32 + 1},
+			name: "negative max pods",
+			goal: GoalState{
+				KubernetesVersion: "1.35.1",
+				MaxPods:           -1,
+				KubeletConfig: KubeletConfig{
+					ImageGCHighThreshold: 85,
+					ImageGCLowThreshold:  80,
+				},
+			},
+			wantErr: "max pods must be positive",
+		},
+		{
+			name: "max pods exceeds int32",
+			goal: GoalState{
+				KubernetesVersion: "1.35.1",
+				MaxPods:           math.MaxInt32 + 1,
+				KubeletConfig: KubeletConfig{
+					ImageGCHighThreshold: 85,
+					ImageGCLowThreshold:  80,
+				},
+			},
 			wantErr: "max pods must be less than or equal to",
 		},
 		{
 			name: "negative image GC high threshold",
 			goal: GoalState{
 				KubernetesVersion: "1.35.1",
+				MaxPods:           110,
 				KubeletConfig: KubeletConfig{
 					ImageGCHighThreshold: -1,
+					ImageGCLowThreshold:  80,
 				},
 			},
-			wantErr: "image GC high threshold must be non-negative",
+			wantErr: "image GC high threshold must be positive",
 		},
 		{
 			name: "negative image GC low threshold",
 			goal: GoalState{
 				KubernetesVersion: "1.35.1",
+				MaxPods:           110,
 				KubeletConfig: KubeletConfig{
-					ImageGCLowThreshold: -1,
+					ImageGCHighThreshold: 85,
+					ImageGCLowThreshold:  -1,
 				},
 			},
 			wantErr: "image GC low threshold must be non-negative",
@@ -283,7 +306,7 @@ func TestGoalStateValidate(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			t.Parallel()
 
-			err := tt.goal.validate()
+			err := tt.goal.Validate()
 			if tt.wantErr != "" {
 				if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
 					t.Fatalf("validate() error = %v, want containing %q", err, tt.wantErr)
@@ -331,7 +354,8 @@ func TestMachineFromARM(t *testing.T) {
 	if len(machine.Goal.NodeTaints) != 1 || machine.Goal.NodeTaints[0] != "dedicated=flex:NoSchedule" {
 		t.Fatalf("goal taints = %#v", machine.Goal.NodeTaints)
 	}
-	if machine.Goal.KubeletConfig.ImageGCHighThreshold != 85 || machine.Goal.KubeletConfig.ImageGCLowThreshold != 80 {
+	if machine.Goal.KubeletConfig.ImageGCHighThreshold != 85 ||
+		machine.Goal.KubeletConfig.ImageGCLowThreshold == nil || *machine.Goal.KubeletConfig.ImageGCLowThreshold != 80 {
 		t.Fatalf("kubelet config = %#v", machine.Goal.KubeletConfig)
 	}
 	if machine.Status.ProvisioningState != ProvisioningStateSucceeded {
@@ -415,6 +439,24 @@ func TestMachineFromARMDoesNotUseCurrentOrchestratorVersionAsGoal(t *testing.T)
 	}
 }
 
+func TestMachineFromARMResolvesMinorVersionAlias(t *testing.T) {
+	t.Parallel()
+
+	machine := machineFromARM(armcontainerservice.Machine{
+		Properties: &armcontainerservice.MachineProperties{
+			ETag: ptr("42"),
+			Kubernetes: &armcontainerservice.MachineKubernetesProfile{
+				OrchestratorVersion:        ptr(" v1.35 "),
+				CurrentOrchestratorVersion: ptr(" v1.35.2 "),
+			},
+		},
+	}, "", "")
+
+	if machine.Goal.KubernetesVersion != "1.35.2" {
+		t.Fatalf("KubernetesVersion = %q, want resolved patch 1.35.2", machine.Goal.KubernetesVersion)
+	}
+}
+
 func TestMachineFromARMDoesNotSynthesizeSettingsVersion(t *testing.T) {
 	t.Parallel()
 

pkg/aksmachine/client_incluster.go

@@ -105,6 +105,9 @@ func clusterEndpointBaseURL(restCfg *rest.Config, endpointURL string) (*url.URL,
 }
 
 func (c *clusterEndpointClient) Create(ctx context.Context, desired GoalState) (*Machine, error) {
+	if err := desired.Validate(); err != nil {
+		return nil, fmt.Errorf("validate goal state: %w", err)
+	}
 	requestURL := c.machineURL(c.nodeName)
 	payload := armcontainerservice.Machine{
 		Properties: &armcontainerservice.MachineProperties{
@@ -130,7 +133,7 @@ func (c *clusterEndpointClient) Create(ctx context.Context, desired GoalState) (
 
 	if resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusMethodNotAllowed || resp.StatusCode == http.StatusNoContent {
 		c.logger.Debug("cluster endpoint did not apply machine create request; verifying pre-created machine", "status", resp.Status)
-		return c.adoptExistingMachine(ctx, desired)
+		return c.adoptExistingMachine(ctx)
 	}
 	if resp.StatusCode < 200 || resp.StatusCode > 299 {
 		return nil, clusterEndpointHTTPError("create machine through cluster endpoint", requestURL, resp)
@@ -140,7 +143,7 @@ func (c *clusterEndpointClient) Create(ctx context.Context, desired GoalState) (
 		return nil, fmt.Errorf("read cluster endpoint machine create response: %w", err)
 	}
 	if strings.TrimSpace(string(data)) == "" {
-		return c.adoptExistingMachine(ctx, desired)
+		return c.adoptExistingMachine(ctx)
 	}
 	machine, err := machineFromEndpointJSON(data)
 	if err != nil {
@@ -152,33 +155,17 @@ func (c *clusterEndpointClient) Create(ctx context.Context, desired GoalState) (
 	if machine.Name == "" {
 		machine.Name = c.nodeName
 	}
-	if err := validateAdoptedMachine(machine, desired); err != nil {
-		return nil, err
-	}
 	return machine, nil
 }
 
-func (c *clusterEndpointClient) adoptExistingMachine(ctx context.Context, desired GoalState) (*Machine, error) {
+func (c *clusterEndpointClient) adoptExistingMachine(ctx context.Context) (*Machine, error) {
 	machine, err := c.Get(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("verify pre-created machine from cluster endpoint: %w", err)
 	}
-	if err := validateAdoptedMachine(machine, desired); err != nil {
-		return nil, err
-	}
 	return machine, nil
 }
 
-func validateAdoptedMachine(machine *Machine, desired GoalState) error {
-	if err := machine.Validate(); err != nil {
-		return fmt.Errorf("cluster endpoint returned invalid machine: %w", err)
-	}
-	if desired.KubernetesVersion != "" && machine.Goal.KubernetesVersion != "" && machine.Goal.KubernetesVersion != desired.KubernetesVersion {
-		return fmt.Errorf("pre-created machine Kubernetes version %q does not match desired %q", machine.Goal.KubernetesVersion, desired.KubernetesVersion)
-	}
-	return nil
-}
-
 func (c *clusterEndpointClient) Get(ctx context.Context) (*Machine, error) {
 	requestURL := c.machineURL(c.nodeName)
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, requestURL.String(), nil)