OSAC-5159: [DEV] Operator auto-detects subnet count and skips k8s manager

osac-operator/internal/controller/constants_common.go

@@ -43,6 +43,11 @@ const (
 	// already be gone at delete time.
 	osacK8sImplementationStrategyAnnotation = osacPrefix + "/k8s-implementation-strategy"
 
+	// osacSkipK8sManagerAnnotation explicitly suppresses the K8s manager for a Subnet.
+	// The Subnet controller honors it only for cudn_evpn, the Phase 1 manager that
+	// supports one CUDN per VirtualNetwork.
+	osacSkipK8sManagerAnnotation = osacPrefix + "/skip-k8s-manager"
+
 	// osacExternalIPPoolNameAnnotation is the K8s resource name of the parent ExternalIPPool.
 	// set on ExternalIP CRs
 	osacExternalIPPoolNameAnnotation = osacPrefix + "/externalippool-name"

osac-operator/internal/controller/subnet_controller.go

@@ -19,6 +19,7 @@ package controller
 import (
 	"context"
 	"fmt"
+	"sort"
 	"time"
 
 	coordinationv1 "k8s.io/api/coordination/v1"
@@ -257,6 +258,10 @@ func (r *SubnetReconciler) handleUpdate(ctx context.Context, subnet *v1alpha1.Su
 	if err != nil {
 		return ctrl.Result{}, err
 	}
+	plan, err = r.applySequentialProvisioningPolicy(ctx, subnet, plan)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
 	implementationStrategy := vnet.Annotations[osacImplementationStrategyAnnotation]
 	// plan may be nil here (no-dispatcher legacy path); FabricTarget/K8sTarget have
 	// nil-receiver-safe implementations that return nil in that case, so this — unlike
@@ -338,6 +343,95 @@ func (r *SubnetReconciler) handleUpdate(ctx context.Context, subnet *v1alpha1.Su
 	return r.handleProvisioning(ctx, subnet, plan)
 }
 
+// applySequentialProvisioningPolicy limits cudn_evpn to one K8s target per
+// VirtualNetwork. The oldest Subnet keeps K8s provisioning; later Subnets are
+// fabric-only unless they already have K8s target history, which is grandfathered
+// to preserve previously provisioned CUDNs. An explicit skip annotation always
+// removes the K8s target so the normal stale-target deprovisioning path can run.
+func (r *SubnetReconciler) applySequentialProvisioningPolicy(
+	ctx context.Context, subnet *v1alpha1.Subnet, plan *dispatcher.DispatchPlan,
+) (*dispatcher.DispatchPlan, error) {
+	k8sTarget := plan.K8sTarget()
+	if k8sTarget == nil || k8sTarget.Manager.Name != "cudn_evpn" {
+		return plan, nil
+	}
+	if subnet.Annotations[osacSkipK8sManagerAnnotation] == labelValueTrue {
+		return withoutK8sTarget(plan), nil
+	}
+
+	reader := r.APIReader
+	if reader == nil {
+		reader = r.Client
+	}
+	subnetList := &v1alpha1.SubnetList{}
+	if err := reader.List(ctx, subnetList, client.InNamespace(subnet.Namespace)); err != nil {
+		return nil, fmt.Errorf("listing Subnets for sequential cudn_evpn provisioning: %w", err)
+	}
+	candidates := make([]*v1alpha1.Subnet, 0, len(subnetList.Items)+1)
+	foundCurrent := false
+	for i := range subnetList.Items {
+		candidate := &subnetList.Items[i]
+		if candidate.Spec.VirtualNetwork != subnet.Spec.VirtualNetwork {
+			continue
+		}
+		candidates = append(candidates, candidate)
+		if candidate.Name == subnet.Name {
+			foundCurrent = true
+		}
+	}
+	// A newly created Subnet may not yet appear in a cache-backed reader. Include
+	// the reconcile object so selection remains deterministic for back-to-back creates.
+	if !foundCurrent {
+		candidates = append(candidates, subnet)
+	}
+	if oldestOperatorSubnet(candidates).Name == subnet.Name || hasK8sTargetHistory(subnet) {
+		return plan, nil
+	}
+	return withoutK8sTarget(plan), nil
+}
+
+func oldestOperatorSubnet(subnets []*v1alpha1.Subnet) *v1alpha1.Subnet {
+	sort.Slice(subnets, func(i, j int) bool {
+		left, right := subnets[i], subnets[j]
+		switch {
+		case left.CreationTimestamp.IsZero() && !right.CreationTimestamp.IsZero():
+			return true
+		case !left.CreationTimestamp.IsZero() && right.CreationTimestamp.IsZero():
+			return false
+		case !left.CreationTimestamp.Equal(&right.CreationTimestamp):
+			return left.CreationTimestamp.Before(&right.CreationTimestamp)
+		default:
+			return left.Name < right.Name
+		}
+	})
+	return subnets[0]
+}
+
+func hasK8sTargetHistory(subnet *v1alpha1.Subnet) bool {
+	if subnet.Annotations[osacK8sImplementationStrategyAnnotation] != "" {
+		return true
+	}
+	for _, job := range subnet.Status.ProvisioningJobs {
+		if job.Target == string(dispatcher.ManagerRoleK8s) && job.Type == v1alpha1.JobTypeProvision {
+			return true
+		}
+	}
+	return false
+}
+
+func withoutK8sTarget(plan *dispatcher.DispatchPlan) *dispatcher.DispatchPlan {
+	if plan == nil || plan.K8sTarget() == nil {
+		return plan
+	}
+	filtered := &dispatcher.DispatchPlan{Targets: make([]dispatcher.DispatchTarget, 0, len(plan.Targets)-1)}
+	for _, target := range plan.Targets {
+		if target.Role != dispatcher.ManagerRoleK8s {
+			filtered.Targets = append(filtered.Targets, target)
+		}
+	}
+	return filtered
+}
+
 // ensureVNetLockLease creates a K8s Lease for V-Net mutex locking if it
 // doesn't already exist. The Lease is owned by the Subnet CR and will be
 // garbage collected when the Subnet is deleted.

osac-operator/internal/controller/subnet_sequential_provisioning_test.go

@@ -0,0 +1,262 @@
+/*
+Copyright 2026.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+	osacv1alpha1 "github.com/osac-project/osac/osac-operator/api/v1alpha1"
+	"github.com/osac-project/osac/osac-operator/pkg/dispatcher"
+	"github.com/osac-project/osac/osac-operator/pkg/networkmanager"
+)
+
+var _ = Describe("Subnet sequential provisioning policy", func() {
+	var (
+		scheme *runtime.Scheme
+		plan   *dispatcher.DispatchPlan
+	)
+
+	BeforeEach(func() {
+		scheme = runtime.NewScheme()
+		Expect(osacv1alpha1.AddToScheme(scheme)).To(Succeed())
+		plan = &dispatcher.DispatchPlan{Targets: []dispatcher.DispatchTarget{
+			{Role: dispatcher.ManagerRoleFabric, Manager: networkmanager.Manager{Name: "netris"}},
+			{Role: dispatcher.ManagerRoleK8s, Manager: networkmanager.Manager{Name: "cudn_evpn"}},
+		}}
+	})
+
+	It("keeps the oldest Subnet on K8s even when both Subnets exist before reconcile", func() {
+		older := sequentialSubnet("z-older", "vn-a", time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC))
+		newer := sequentialSubnet("a-newer", "vn-a", time.Date(2026, 10, 2, 0, 0, 0, 0, time.UTC))
+		unrelatedNetwork := sequentialSubnet("unrelated", "vn-b", time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC))
+		k8sClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(older, newer, unrelatedNetwork).Build()
+		reconciler := &SubnetReconciler{Client: k8sClient, APIReader: k8sClient}
+
+		result, err := reconciler.applySequentialProvisioningPolicy(context.Background(), newer, plan)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(result.K8sTarget()).To(BeNil())
+		Expect(result.FabricTarget()).NotTo(BeNil())
+		Expect(result.FabricTarget().Manager.Name).To(Equal("netris"))
+
+		result, err = reconciler.applySequentialProvisioningPolicy(context.Background(), older, plan)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(result.K8sTarget()).NotTo(BeNil())
+		Expect(result.FabricTarget()).NotTo(BeNil())
+	})
+
+	It("selects the oldest persisted Subnet from the envtest API server", func() {
+		// Unlike the fake-client cases above, this exercises the policy against the
+		// suite's real Kubernetes API server and its persisted Subnet metadata.
+		suffix := fmt.Sprintf("%x", time.Now().UnixNano())
+		virtualNetworkID := "envtest-vnet-" + suffix
+		tenantID := "envtest-tenant-" + suffix
+		namespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{
+			Name: "envtest-sequential-" + suffix,
+			Annotations: map[string]string{
+				osacTenantKey:                       tenantID,
+				"osac.openshift.io/owner-reference": virtualNetworkID,
+			},
+		}}
+		Expect(k8sClient.Create(context.Background(), namespace)).To(Succeed())
+		DeferCleanup(func() {
+			Expect(client.IgnoreNotFound(k8sClient.Delete(context.Background(), namespace))).To(Succeed())
+		})
+
+		older := sequentialSubnet("z-envtest-older-"+suffix, virtualNetworkID, time.Time{})
+		newer := sequentialSubnet("a-envtest-newer-"+suffix, virtualNetworkID, time.Time{})
+		older.Spec.IPv4CIDR = "10.100.1.0/24"
+		newer.Spec.IPv4CIDR = "10.100.2.0/24"
+		for _, subnet := range []*osacv1alpha1.Subnet{older, newer} {
+			subnet.Namespace = namespace.Name
+			subnet.Annotations = map[string]string{
+				osacTenantKey:                       tenantID,
+				"osac.openshift.io/owner-reference": virtualNetworkID,
+			}
+		}
+
+		Expect(k8sClient.Create(context.Background(), older)).To(Succeed())
+		DeferCleanup(func() {
+			Expect(client.IgnoreNotFound(k8sClient.Delete(context.Background(), older))).To(Succeed())
+		})
+		Expect(k8sClient.Create(context.Background(), newer)).To(Succeed())
+		DeferCleanup(func() {
+			Expect(client.IgnoreNotFound(k8sClient.Delete(context.Background(), newer))).To(Succeed())
+		})
+
+		// Read back server-assigned creation timestamps. If the API server records
+		// both creates in the same timestamp tick, the controller's name tie-breaker
+		// determines the expected oldest Subnet.
+		persisted := []*osacv1alpha1.Subnet{{}, {}}
+		Expect(k8sClient.Get(context.Background(), client.ObjectKeyFromObject(older), persisted[0])).To(Succeed())
+		Expect(k8sClient.Get(context.Background(), client.ObjectKeyFromObject(newer), persisted[1])).To(Succeed())
+		expectedOldest := persisted[0]
+		if persisted[1].CreationTimestamp.Before(&persisted[0].CreationTimestamp) ||
+			(persisted[1].CreationTimestamp.Equal(&persisted[0].CreationTimestamp) && persisted[1].Name < persisted[0].Name) {
+			expectedOldest = persisted[1]
+		}
+
+		reconciler := &SubnetReconciler{Client: k8sClient, APIReader: k8sClient}
+		for _, candidate := range persisted {
+			selectedPlan, err := reconciler.applySequentialProvisioningPolicy(context.Background(), candidate, plan)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(selectedPlan.FabricTarget()).NotTo(BeNil())
+			if candidate.Name == expectedOldest.Name {
+				Expect(selectedPlan.K8sTarget()).NotTo(BeNil(), candidate.Name)
+			} else {
+				Expect(selectedPlan.K8sTarget()).To(BeNil(), candidate.Name)
+			}
+		}
+	})
+
+	It("uses Subnet name as a stable tie-breaker for equal creation timestamps", func() {
+		created := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)
+		nameLater := sequentialSubnet("z-name-later", "vn-a", created)
+		nameEarlier := sequentialSubnet("a-name-earlier", "vn-a", created)
+		k8sClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(nameLater, nameEarlier).Build()
+		reconciler := &SubnetReconciler{Client: k8sClient, APIReader: k8sClient}
+
+		result, err := reconciler.applySequentialProvisioningPolicy(context.Background(), nameLater, plan)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(result.K8sTarget()).To(BeNil())
+
+		result, err = reconciler.applySequentialProvisioningPolicy(context.Background(), nameEarlier, plan)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(result.K8sTarget()).NotTo(BeNil())
+	})
+
+	It("includes the current Subnet when it is missing from the APIReader list", func() {
+		current := sequentialSubnet("a-current-oldest", "vn-a", time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC))
+		sibling := sequentialSubnet("z-listed-later", "vn-a", time.Date(2026, 10, 2, 0, 0, 0, 0, time.UTC))
+		k8sClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(sibling).Build()
+		reconciler := &SubnetReconciler{Client: k8sClient, APIReader: k8sClient}
+
+		result, err := reconciler.applySequentialProvisioningPolicy(context.Background(), current, plan)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(result.K8sTarget()).NotTo(BeNil())
+	})
+
+	It("honors an explicit skip annotation over previous K8s target history", func() {
+		older := sequentialSubnet("older", "vn-a", time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC))
+		later := sequentialSubnet("later", "vn-a", time.Date(2026, 10, 2, 0, 0, 0, 0, time.UTC))
+		later.Annotations = map[string]string{
+			osacSkipK8sManagerAnnotation:            labelValueTrue,
+			osacK8sImplementationStrategyAnnotation: "cudn_evpn",
+		}
+		later.Status.ProvisioningJobs = []osacv1alpha1.JobStatus{{
+			JobID:     "k8s-provision-job",
+			Type:      osacv1alpha1.JobTypeProvision,
+			Timestamp: metav1.NewTime(time.Date(2026, 10, 2, 0, 0, 0, 0, time.UTC)),
+			State:     osacv1alpha1.JobStateSucceeded,
+			Target:    string(dispatcher.ManagerRoleK8s),
+		}}
+		k8sClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(older, later).Build()
+		reconciler := &SubnetReconciler{Client: k8sClient, APIReader: k8sClient}
+
+		result, err := reconciler.applySequentialProvisioningPolicy(context.Background(), later, plan)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(result.K8sTarget()).To(BeNil())
+		Expect(result.FabricTarget()).NotTo(BeNil())
+	})
+
+	It("preserves a later Subnet's previously provisioned K8s target", func() {
+		older := sequentialSubnet("older", "vn-a", time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC))
+		later := sequentialSubnet("later", "vn-a", time.Date(2026, 10, 2, 0, 0, 0, 0, time.UTC))
+		later.Annotations = map[string]string{osacK8sImplementationStrategyAnnotation: "cudn_evpn"}
+		k8sClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(older, later).Build()
+		reconciler := &SubnetReconciler{Client: k8sClient, APIReader: k8sClient}
+
+		result, err := reconciler.applySequentialProvisioningPolicy(context.Background(), later, plan)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(result.K8sTarget()).NotTo(BeNil())
+	})
+
+	It("preserves a later Subnet with K8s provisioning job history", func() {
+		older := sequentialSubnet("older", "vn-a", time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC))
+		later := sequentialSubnet("later", "vn-a", time.Date(2026, 10, 2, 0, 0, 0, 0, time.UTC))
+		later.Status.ProvisioningJobs = []osacv1alpha1.JobStatus{{
+			JobID:     "k8s-provision-job",
+			Type:      osacv1alpha1.JobTypeProvision,
+			Timestamp: metav1.NewTime(time.Date(2026, 10, 2, 0, 0, 0, 0, time.UTC)),
+			State:     osacv1alpha1.JobStateSucceeded,
+			Target:    string(dispatcher.ManagerRoleK8s),
+		}}
+		k8sClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(older, later).Build()
+		reconciler := &SubnetReconciler{Client: k8sClient, APIReader: k8sClient}
+
+		result, err := reconciler.applySequentialProvisioningPolicy(context.Background(), later, plan)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(result.K8sTarget()).NotTo(BeNil())
+	})
+
+	It("returns an error when listing Subnets fails", func() {
+		current := sequentialSubnet("current", "vn-a", time.Now())
+		k8sClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(current).Build()
+		reconciler := &SubnetReconciler{
+			Client:    k8sClient,
+			APIReader: sequentialListErrorReader{Reader: k8sClient, err: errors.New("APIReader unavailable")},
+		}
+
+		_, err := reconciler.applySequentialProvisioningPolicy(context.Background(), current, plan)
+		Expect(err).To(HaveOccurred())
+		Expect(err.Error()).To(ContainSubstring("APIReader unavailable"))
+	})
+
+	It("leaves other K8s managers unchanged", func() {
+		later := sequentialSubnet("later", "vn-a", time.Now())
+		later.Annotations = map[string]string{osacSkipK8sManagerAnnotation: labelValueTrue}
+		otherManagerPlan := &dispatcher.DispatchPlan{Targets: []dispatcher.DispatchTarget{
+			plan.Targets[0],
+			{Role: dispatcher.ManagerRoleK8s, Manager: networkmanager.Manager{Name: "cudn_net"}},
+		}}
+		reconciler := &SubnetReconciler{}
+
+		result, err := reconciler.applySequentialProvisioningPolicy(context.Background(), later, otherManagerPlan)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(result.K8sTarget()).NotTo(BeNil())
+	})
+})
+
+func sequentialSubnet(name, virtualNetwork string, created time.Time) *osacv1alpha1.Subnet {
+	return &osacv1alpha1.Subnet{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:              name,
+			Namespace:         "osac",
+			CreationTimestamp: metav1.NewTime(created),
+		},
+		Spec: osacv1alpha1.SubnetSpec{VirtualNetwork: virtualNetwork},
+	}
+}
+
+type sequentialListErrorReader struct {
+	client.Reader
+	err error
+}
+
+func (r sequentialListErrorReader) List(context.Context, client.ObjectList, ...client.ListOption) error {
+	return r.err
+}