fix: Reprogram NCs missing from CNS for Succeeded CRs after restart

.pipelines/build/dockerfiles/azure-iptables-monitor.Dockerfile

@@ -3,10 +3,10 @@
 ARG ARCH
 
 # mcr.microsoft.com/azurelinux/base/core:3.0
-FROM mcr.microsoft.com/azurelinux/base/core@sha256:a452d39c91576f5a2c983c7d3b62521fabd08e16b4a7237e24bf2be3b06e1651 AS mariner-core
+FROM mcr.microsoft.com/azurelinux/base/core@sha256:35149ae8dd179684f969944f54a337c665a64e702486154eb44253fb39c2505b AS mariner-core
 
 # mcr.microsoft.com/azurelinux/distroless/minimal:3.0
-FROM mcr.microsoft.com/azurelinux/distroless/minimal@sha256:22810fd97d6ad5ec7d5bdd5b00233a3050be01d9e26b47b16cb6f1a7f178834b AS mariner-distroless
+FROM mcr.microsoft.com/azurelinux/distroless/minimal@sha256:5a66f9f16ac675db2a8229dac72d83811b73b502d6ad192d8b374c7f3be498af AS mariner-distroless
 
 FROM mariner-core AS iptools
 RUN tdnf install -y iptables iproute

.pipelines/build/dockerfiles/cns.Dockerfile

@@ -11,11 +11,11 @@ ENTRYPOINT ["azure-cns.exe"]
 EXPOSE 10090
 
 # mcr.microsoft.com/azurelinux/base/core:3.0
-FROM --platform=linux/${ARCH} mcr.microsoft.com/azurelinux/base/core@sha256:a452d39c91576f5a2c983c7d3b62521fabd08e16b4a7237e24bf2be3b06e1651 AS build-helper
+FROM --platform=linux/${ARCH} mcr.microsoft.com/azurelinux/base/core@sha256:35149ae8dd179684f969944f54a337c665a64e702486154eb44253fb39c2505b AS build-helper
 RUN tdnf install -y iptables
 
 # mcr.microsoft.com/azurelinux/distroless/minimal:3.0
-FROM --platform=linux/${ARCH} mcr.microsoft.com/azurelinux/distroless/minimal@sha256:22810fd97d6ad5ec7d5bdd5b00233a3050be01d9e26b47b16cb6f1a7f178834b AS linux
+FROM --platform=linux/${ARCH} mcr.microsoft.com/azurelinux/distroless/minimal@sha256:5a66f9f16ac675db2a8229dac72d83811b73b502d6ad192d8b374c7f3be498af AS linux
 ARG ARTIFACT_DIR .
 
 COPY --from=build-helper /usr/sbin/*tables* /usr/sbin/

azure-ipam/Dockerfile

@@ -9,7 +9,7 @@ ARG OS
 FROM --platform=linux/${ARCH} mcr.microsoft.com/oss/go/microsoft/golang@sha256:bc7423b52b62e8f0281b5f7f564eb1862dc315bc57e1373c6a81e87ef3ac39ab AS go
 
 # mcr.microsoft.com/azurelinux/base/core:3.0
-FROM --platform=linux/${ARCH} mcr.microsoft.com/azurelinux/base/core@sha256:a452d39c91576f5a2c983c7d3b62521fabd08e16b4a7237e24bf2be3b06e1651 AS mariner-core
+FROM --platform=linux/${ARCH} mcr.microsoft.com/azurelinux/base/core@sha256:35149ae8dd179684f969944f54a337c665a64e702486154eb44253fb39c2505b AS mariner-core
 
 FROM go AS azure-ipam
 ARG OS

azure-iptables-monitor/Dockerfile

@@ -3,10 +3,10 @@
 ARG ARCH
 
 # mcr.microsoft.com/azurelinux/base/core:3.0
-FROM mcr.microsoft.com/azurelinux/base/core@sha256:a452d39c91576f5a2c983c7d3b62521fabd08e16b4a7237e24bf2be3b06e1651 AS mariner-core
+FROM mcr.microsoft.com/azurelinux/base/core@sha256:35149ae8dd179684f969944f54a337c665a64e702486154eb44253fb39c2505b AS mariner-core
 
 # mcr.microsoft.com/azurelinux/distroless/minimal:3.0
-FROM mcr.microsoft.com/azurelinux/distroless/minimal@sha256:22810fd97d6ad5ec7d5bdd5b00233a3050be01d9e26b47b16cb6f1a7f178834b AS mariner-distroless
+FROM mcr.microsoft.com/azurelinux/distroless/minimal@sha256:5a66f9f16ac675db2a8229dac72d83811b73b502d6ad192d8b374c7f3be498af AS mariner-distroless
 
 # mcr.microsoft.com/oss/go/microsoft/golang:1.24-azurelinux3.0
 FROM --platform=linux/${ARCH} mcr.microsoft.com/oss/go/microsoft/golang@sha256:bc7423b52b62e8f0281b5f7f564eb1862dc315bc57e1373c6a81e87ef3ac39ab AS go

cni/Dockerfile

@@ -9,7 +9,7 @@ ARG OS
 FROM --platform=linux/${ARCH} mcr.microsoft.com/oss/go/microsoft/golang@sha256:bc7423b52b62e8f0281b5f7f564eb1862dc315bc57e1373c6a81e87ef3ac39ab AS go
 
 # mcr.microsoft.com/azurelinux/base/core:3.0
-FROM --platform=linux/${ARCH} mcr.microsoft.com/azurelinux/base/core@sha256:a452d39c91576f5a2c983c7d3b62521fabd08e16b4a7237e24bf2be3b06e1651 AS mariner-core
+FROM --platform=linux/${ARCH} mcr.microsoft.com/azurelinux/base/core@sha256:35149ae8dd179684f969944f54a337c665a64e702486154eb44253fb39c2505b AS mariner-core
 
 FROM go AS azure-vnet
 ARG OS

cns/Dockerfile

@@ -8,10 +8,10 @@ ARG OS
 FROM --platform=linux/${ARCH} mcr.microsoft.com/oss/go/microsoft/golang@sha256:bc7423b52b62e8f0281b5f7f564eb1862dc315bc57e1373c6a81e87ef3ac39ab AS go
 
 # mcr.microsoft.com/azurelinux/base/core:3.0
-FROM mcr.microsoft.com/azurelinux/base/core@sha256:a452d39c91576f5a2c983c7d3b62521fabd08e16b4a7237e24bf2be3b06e1651 AS mariner-core
+FROM mcr.microsoft.com/azurelinux/base/core@sha256:35149ae8dd179684f969944f54a337c665a64e702486154eb44253fb39c2505b AS mariner-core
 
 # mcr.microsoft.com/azurelinux/distroless/minimal:3.0
-FROM mcr.microsoft.com/azurelinux/distroless/minimal@sha256:22810fd97d6ad5ec7d5bdd5b00233a3050be01d9e26b47b16cb6f1a7f178834b AS mariner-distroless
+FROM mcr.microsoft.com/azurelinux/distroless/minimal@sha256:5a66f9f16ac675db2a8229dac72d83811b73b502d6ad192d8b374c7f3be498af AS mariner-distroless
 
 FROM --platform=linux/${ARCH} go AS builder
 ARG OS

cns/multitenantcontroller/multitenantoperator/multitenantcrdreconciler.go

@@ -87,9 +87,11 @@ func (r *multiTenantCrdReconciler) Reconcile(ctx context.Context, request reconc
 		return ctrl.Result{}, nil
 	}
 
-	// Do nothing if the network container hasn't been initialized yet from control plane.
-	if nc.Status.State != NCStateInitialized {
-		logger.Printf("MultiTenantNetworkContainer %s hasn't initialized yet, skip reconciling", request.NamespacedName.String())
+	// Skip reconciliation for CRs that are not yet initialized and not previously succeeded.
+	// For "Succeeded" CRs, we must verify the NC still exists in CNS — it may have been lost
+	// due to CNS restart or state file corruption. If missing, we fall through to reprogram it.
+	if nc.Status.State != NCStateInitialized && nc.Status.State != NCStateSucceeded {
+		logger.Printf("MultiTenantNetworkContainer %s in state %s, skip reconciling", request.NamespacedName.String(), nc.Status.State) //nolint:staticcheck // TODO: migrate to cns/logger/v2
 		return ctrl.Result{}, nil
 	}
 
@@ -122,6 +124,10 @@ func (r *multiTenantCrdReconciler) Reconcile(ctx context.Context, request reconc
 		return ctrl.Result{}, err
 	}
 
+	if nc.Status.State == NCStateSucceeded {
+		logger.Warnf("NC %s (UUID: %s) missing from CNS despite CR state Succeeded, reprogramming", request.NamespacedName.String(), nc.Spec.UUID) //nolint:staticcheck // TODO: migrate to cns/logger/v2
+	}
+
 	// Check that the MultiTenantInfo is set
 	if reflect.DeepEqual(ncapi.MultiTenantInfo{}, nc.Status.MultiTenantInfo) {
 		logger.Errorf("expected NC status multitenant info to not be empty for object %s", request.NamespacedName)

cns/multitenantcontroller/multitenantoperator/multitenantcrdreconciler_test.go

@@ -114,6 +114,139 @@ var _ = Describe("multiTenantCrdReconciler", func() {
 			Expect(err).To(BeNil())
 		})
 
+		It("Should reprogram NC when CR is in Succeeded state but NC is missing from CNS", func() {
+			uuid := uuidValue
+			nc := ncapi.MultiTenantNetworkContainer{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      namespacedName.Name,
+					Namespace: namespacedName.Namespace,
+				},
+				Spec: ncapi.MultiTenantNetworkContainerSpec{
+					UUID: uuid,
+				},
+				Status: ncapi.MultiTenantNetworkContainerStatus{
+					State: NCStateSucceeded,
+					MultiTenantInfo: ncapi.MultiTenantInfo{
+						EncapType: "Vlan",
+						ID:        1,
+					},
+					IPSubnet: "10.0.0.0/8",
+					IP:       "10.1.0.0",
+				},
+			}
+
+			orchestratorContext, err := json.Marshal(podInfo)
+			Expect(err).To(BeNil())
+
+			kubeClient.EXPECT().Get(gomock.Any(), namespacedName, gomock.Any()).SetArg(2, nc)
+			// NC is missing from CNS — returns UnknownContainerID
+			cnsRestService.EXPECT().GetNetworkContainerInternal(cns.GetNetworkContainerRequest{
+				NetworkContainerid:  uuid,
+				OrchestratorContext: orchestratorContext,
+			}).Return(cns.GetNetworkContainerResponse{}, cnstypes.UnknownContainerID)
+
+			// Should reprogram the NC
+			cnsRestService.EXPECT().CreateOrUpdateNetworkContainerInternal(&cns.CreateNetworkContainerRequest{
+				NetworkContainerid:   nc.Spec.UUID,
+				OrchestratorContext:  orchestratorContext,
+				NetworkContainerType: cns.Kubernetes,
+				Version:              "0",
+				IPConfiguration: cns.IPConfiguration{
+					IPSubnet: cns.IPSubnet{
+						IPAddress:    nc.Status.IP,
+						PrefixLength: 8,
+					},
+					GatewayIPAddress: nc.Status.Gateway,
+				},
+				PrimaryInterfaceIdentifier: nc.Status.PrimaryInterfaceIdentifier,
+				MultiTenancyInfo: cns.MultiTenancyInfo{
+					EncapType: nc.Status.MultiTenantInfo.EncapType,
+					ID:        int(nc.Status.MultiTenantInfo.ID),
+				},
+			}).Return(cnstypes.Success)
+
+			kubeClient.EXPECT().Status().DoAndReturn(func() client.SubResourceWriter {
+				nc.Status.State = NCStateSucceeded
+				statusWriter.EXPECT().Update(gomock.Any(), gomock.Any()).SetArg(1, nc)
+				return statusWriter
+			})
+
+			_, err = reconciler.Reconcile(context.TODO(), reconcile.Request{
+				NamespacedName: namespacedName,
+			})
+			Expect(err).To(BeNil())
+		})
+
+		It("Should skip reconciliation when CR is in Succeeded state and NC exists in CNS", func() {
+			uuid := uuidValue
+			nc := ncapi.MultiTenantNetworkContainer{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      namespacedName.Name,
+					Namespace: namespacedName.Namespace,
+				},
+				Spec: ncapi.MultiTenantNetworkContainerSpec{
+					UUID: uuid,
+				},
+				Status: ncapi.MultiTenantNetworkContainerStatus{
+					State: NCStateSucceeded,
+					MultiTenantInfo: ncapi.MultiTenantInfo{
+						EncapType: "Vlan",
+						ID:        1,
+					},
+				},
+			}
+
+			orchestratorContext, err := json.Marshal(podInfo)
+			Expect(err).To(BeNil())
+
+			kubeClient.EXPECT().Get(gomock.Any(), namespacedName, gomock.Any()).SetArg(2, nc)
+			// NC exists in CNS — should skip without reprogramming
+			cnsRestService.EXPECT().GetNetworkContainerInternal(cns.GetNetworkContainerRequest{
+				NetworkContainerid:  uuid,
+				OrchestratorContext: orchestratorContext,
+			}).Return(cns.GetNetworkContainerResponse{}, cnstypes.Success)
+
+			_, err = reconciler.Reconcile(context.TODO(), reconcile.Request{
+				NamespacedName: namespacedName,
+			})
+			Expect(err).To(BeNil())
+		})
+
+		It("Should return error when CR is in Succeeded state and CNS returns transient error", func() {
+			uuid := uuidValue
+			nc := ncapi.MultiTenantNetworkContainer{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      namespacedName.Name,
+					Namespace: namespacedName.Namespace,
+				},
+				Spec: ncapi.MultiTenantNetworkContainerSpec{
+					UUID: uuid,
+				},
+				Status: ncapi.MultiTenantNetworkContainerStatus{
+					State: NCStateSucceeded,
+					MultiTenantInfo: ncapi.MultiTenantInfo{
+						EncapType: "Vlan",
+						ID:        1,
+					},
+				},
+			}
+
+			orchestratorContext, err := json.Marshal(podInfo)
+			Expect(err).To(BeNil())
+
+			kubeClient.EXPECT().Get(gomock.Any(), namespacedName, gomock.Any()).SetArg(2, nc)
+			// CNS returns an internal error — should NOT reprogram, should surface error
+			cnsRestService.EXPECT().GetNetworkContainerInternal(cns.GetNetworkContainerRequest{
+				NetworkContainerid:  uuid,
+				OrchestratorContext: orchestratorContext,
+			}).Return(cns.GetNetworkContainerResponse{}, cnstypes.UnexpectedError)
+
+			_, err = reconciler.Reconcile(context.TODO(), reconcile.Request{
+				NamespacedName: namespacedName,
+			})
+			Expect(err).NotTo(BeNil())
+		})
+
 		It("Should succeed when the NC is in Initialized state and it has not yet been persisted in CNS", func() {
 			uuid := uuidValue
 			var nc ncapi.MultiTenantNetworkContainer = ncapi.MultiTenantNetworkContainer{