diff --git a/.github/workflows/e2e-matrix.yaml b/.github/workflows/e2e-matrix.yaml index fba7f9d5b31a..a4f39bdd266b 100644 --- a/.github/workflows/e2e-matrix.yaml +++ b/.github/workflows/e2e-matrix.yaml @@ -66,7 +66,10 @@ jobs: - name: AMI region: ${{ inputs.region }} source: aws - - name: Scheduling + - name: Scheduling-BestEffort + region: ${{ inputs.region }} + source: aws + - name: Scheduling-Strict region: ${{ inputs.region }} source: aws - name: Storage diff --git a/test/suites/scheduling-besteffort/suite_test.go b/test/suites/scheduling-besteffort/suite_test.go new file mode 100644 index 000000000000..12e05dc94f6b --- /dev/null +++ b/test/suites/scheduling-besteffort/suite_test.go @@ -0,0 +1,48 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scheduling_besteffort_test + +import ( + "testing" + + "sigs.k8s.io/karpenter/pkg/operator/options" + + environmentaws "github.com/aws/karpenter-provider-aws/test/pkg/environment/aws" + "github.com/aws/karpenter-provider-aws/test/suites/scheduling" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestSchedulingBestEffort(t *testing.T) { + RegisterFailHandler(Fail) + BeforeSuite(func() { + scheduling.Env = environmentaws.NewEnvironment(t) + }) + AfterSuite(func() { + scheduling.Env.Stop() + }) + RunSpecs(t, "SchedulingBestEffort") +} + +var _ = BeforeEach(func() { + scheduling.Env.BeforeEach() + scheduling.NodeClass = scheduling.Env.DefaultEC2NodeClass() + scheduling.NodePool = scheduling.Env.DefaultNodePool(scheduling.NodeClass) +}) +var _ = AfterEach(func() { scheduling.Env.Cleanup() }) +var _ = AfterEach(func() { scheduling.Env.AfterEach() }) + +var _ = scheduling.RegisterTests(options.MinValuesPolicyBestEffort) diff --git a/test/suites/scheduling-strict/suite_test.go b/test/suites/scheduling-strict/suite_test.go new file mode 100644 index 000000000000..27fae33fa7cf --- /dev/null +++ b/test/suites/scheduling-strict/suite_test.go @@ -0,0 +1,48 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scheduling_strict_test + +import ( + "testing" + + "sigs.k8s.io/karpenter/pkg/operator/options" + + environmentaws "github.com/aws/karpenter-provider-aws/test/pkg/environment/aws" + "github.com/aws/karpenter-provider-aws/test/suites/scheduling" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestSchedulingStrict(t *testing.T) { + RegisterFailHandler(Fail) + BeforeSuite(func() { + scheduling.Env = environmentaws.NewEnvironment(t) + }) + AfterSuite(func() { + scheduling.Env.Stop() + }) + RunSpecs(t, "SchedulingStrict") +} + +var _ = BeforeEach(func() { + scheduling.Env.BeforeEach() + scheduling.NodeClass = scheduling.Env.DefaultEC2NodeClass() + scheduling.NodePool = scheduling.Env.DefaultNodePool(scheduling.NodeClass) +}) +var _ = AfterEach(func() { scheduling.Env.Cleanup() }) +var _ = AfterEach(func() { scheduling.Env.AfterEach() }) + +var _ = scheduling.RegisterTests(options.MinValuesPolicyStrict) diff --git a/test/suites/scheduling/scheduling.go b/test/suites/scheduling/scheduling.go new file mode 100644 index 000000000000..4ec8d54fd443 --- /dev/null +++ b/test/suites/scheduling/scheduling.go @@ -0,0 +1,1154 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scheduling + +import ( + "fmt" + "os" + "strconv" + + "time" + + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + + "github.com/awslabs/operatorpkg/object" + "github.com/samber/lo" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/sets" + + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" + "sigs.k8s.io/karpenter/pkg/apis/v1alpha1" + "sigs.k8s.io/karpenter/pkg/operator/options" + "sigs.k8s.io/karpenter/pkg/test" + + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" + "github.com/aws/karpenter-provider-aws/test/pkg/debug" + environmentaws "github.com/aws/karpenter-provider-aws/test/pkg/environment/aws" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var Env *environmentaws.Environment +var NodeClass *v1.EC2NodeClass +var NodePool *karpv1.NodePool + +//nolint:gocyclo +func RegisterTests(minValuesPolicy options.MinValuesPolicy) bool { + return Describe("Scheduling", Ordered, ContinueOnFailure, func() { + var selectors sets.Set[string] + + BeforeEach(func() { + // Make the NodePool requirements fully flexible, so we can match well-known label keys + NodePool = test.ReplaceRequirements(NodePool, + karpv1.NodeSelectorRequirementWithMinValues{ + Key: v1.LabelInstanceCategory, + Operator: corev1.NodeSelectorOpExists, + }, + karpv1.NodeSelectorRequirementWithMinValues{ + Key: v1.LabelInstanceGeneration, + Operator: corev1.NodeSelectorOpExists, + }, + ) + Env.ExpectSettingsOverridden(corev1.EnvVar{Name: "MIN_VALUES_POLICY", Value: string(minValuesPolicy)}) + }) + BeforeAll(func() { + selectors = sets.New[string]() + }) + AfterAll(func() { + // Ensure that we're exercising all well known labels + Expect(lo.Keys(selectors)).To(ContainElements(append(karpv1.WellKnownLabels.UnsortedList(), lo.Keys(karpv1.NormalizedLabels)...))) + }) + + It("should apply annotations to the node", func() { + NodePool.Spec.Template.Annotations = map[string]string{ + "foo": "bar", + karpv1.DoNotDisruptAnnotationKey: "true", + } + pod := test.Pod() + Env.ExpectCreated(NodeClass, NodePool, pod) + Env.EventuallyExpectHealthy(pod) + Env.ExpectCreatedNodeCount("==", 1) + Expect(Env.GetNode(pod.Spec.NodeName).Annotations).To(And(HaveKeyWithValue("foo", "bar"), HaveKeyWithValue(karpv1.DoNotDisruptAnnotationKey, "true"))) + }) + + Context("Labels", func() { + It("should support well-known labels for instance type selection", func() { + nodeSelector := map[string]string{ + // Well Known + karpv1.NodePoolLabelKey: NodePool.Name, + corev1.LabelInstanceTypeStable: "c5.large", + // Well Known to AWS + v1.LabelInstanceHypervisor: "nitro", + v1.LabelInstanceCategory: "c", + v1.LabelInstanceGeneration: "5", + v1.LabelInstanceFamily: "c5", + v1.LabelInstanceSize: "large", + v1.LabelInstanceCPU: "2", + v1.LabelInstanceCPUManufacturer: "intel", + v1.LabelInstanceCPUSustainedClockSpeedMhz: "3400", + v1.LabelInstanceMemory: "4096", + v1.LabelInstanceEBSBandwidth: "4750", + v1.LabelInstanceNetworkBandwidth: "750", + } + selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels + requirements := lo.MapToSlice(nodeSelector, func(key string, value string) corev1.NodeSelectorRequirement { + return corev1.NodeSelectorRequirement{Key: key, Operator: corev1.NodeSelectorOpIn, Values: []string{value}} + }) + deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ + NodeSelector: nodeSelector, + NodePreferences: requirements, + NodeRequirements: requirements, + }}) + Env.ExpectCreated(NodeClass, NodePool, deployment) + Env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) + Env.ExpectCreatedNodeCount("==", 1) + }) + It("should support well-known labels for zone id selection", func() { + selectors.Insert(v1.LabelTopologyZoneID) // Add node selector keys to selectors used in testing to ensure we test all labels + deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ + NodeRequirements: []corev1.NodeSelectorRequirement{ + { + Key: v1.LabelTopologyZoneID, + Operator: corev1.NodeSelectorOpIn, + Values: []string{Env.GetSubnetInfo(map[string]string{"karpenter.sh/discovery": Env.ClusterName})[0].ZoneID}, + }, + }, + }}) + Env.ExpectCreated(NodeClass, NodePool, deployment) + Env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) + Env.ExpectCreatedNodeCount("==", 1) + }) + It("should support well-known labels for local NVME storage", func() { + selectors.Insert(v1.LabelInstanceLocalNVME) // Add node selector keys to selectors used in testing to ensure we test all labels + deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ + NodePreferences: []corev1.NodeSelectorRequirement{ + { + Key: v1.LabelInstanceLocalNVME, + Operator: corev1.NodeSelectorOpGt, + Values: []string{"0"}, + }, + }, + NodeRequirements: []corev1.NodeSelectorRequirement{ + { + Key: v1.LabelInstanceLocalNVME, + Operator: corev1.NodeSelectorOpGt, + Values: []string{"0"}, + }, + }, + }}) + Env.ExpectCreated(NodeClass, NodePool, deployment) + Env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) + Env.ExpectCreatedNodeCount("==", 1) + }) + It("should support well-known labels for encryption in transit", func() { + selectors.Insert(v1.LabelInstanceEncryptionInTransitSupported) // Add node selector keys to selectors used in testing to ensure we test all labels + deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ + NodePreferences: []corev1.NodeSelectorRequirement{ + { + Key: v1.LabelInstanceEncryptionInTransitSupported, + Operator: corev1.NodeSelectorOpIn, + Values: []string{"true"}, + }, + }, + NodeRequirements: []corev1.NodeSelectorRequirement{ + { + Key: v1.LabelInstanceEncryptionInTransitSupported, + Operator: corev1.NodeSelectorOpIn, + Values: []string{"true"}, + }, + }, + }}) + Env.ExpectCreated(NodeClass, NodePool, deployment) + Env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) + Env.ExpectCreatedNodeCount("==", 1) + }) + It("should support well-known deprecated labels", func() { + nodeSelector := map[string]string{ + // Deprecated Labels + corev1.LabelFailureDomainBetaRegion: Env.Region, + corev1.LabelFailureDomainBetaZone: fmt.Sprintf("%sa", Env.Region), + "topology.ebs.csi.aws.com/zone": fmt.Sprintf("%sa", Env.Region), + + "beta.kubernetes.io/arch": "amd64", + "beta.kubernetes.io/os": "linux", + corev1.LabelInstanceType: "c5.large", + } + selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels + requirements := lo.MapToSlice(nodeSelector, func(key string, value string) corev1.NodeSelectorRequirement { + return corev1.NodeSelectorRequirement{Key: key, Operator: corev1.NodeSelectorOpIn, Values: []string{value}} + }) + deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ + NodeSelector: nodeSelector, + NodePreferences: requirements, + NodeRequirements: requirements, + }}) + Env.ExpectCreated(NodeClass, NodePool, deployment) + Env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) + Env.ExpectCreatedNodeCount("==", 1) + }) + It("should support well-known labels for topology and architecture", func() { + nodeSelector := map[string]string{ + // Well Known + karpv1.NodePoolLabelKey: NodePool.Name, + corev1.LabelTopologyRegion: Env.Region, + corev1.LabelTopologyZone: fmt.Sprintf("%sa", Env.Region), + corev1.LabelOSStable: "linux", + corev1.LabelArchStable: "amd64", + karpv1.CapacityTypeLabelKey: karpv1.CapacityTypeOnDemand, + } + selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels + requirements := lo.MapToSlice(nodeSelector, func(key string, value string) corev1.NodeSelectorRequirement { + return corev1.NodeSelectorRequirement{Key: key, Operator: corev1.NodeSelectorOpIn, Values: []string{value}} + }) + deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ + NodeSelector: nodeSelector, + NodePreferences: requirements, + NodeRequirements: requirements, + }}) + Env.ExpectCreated(NodeClass, NodePool, deployment) + Env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) + Env.ExpectCreatedNodeCount("==", 1) + }) + It("should support well-known labels for a gpu (nvidia)", func() { + nodeSelector := map[string]string{ + v1.LabelInstanceGPUName: "t4", + v1.LabelInstanceGPUMemory: "16384", + v1.LabelInstanceGPUManufacturer: "nvidia", + v1.LabelInstanceGPUCount: "1", + } + selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels + requirements := lo.MapToSlice(nodeSelector, func(key string, value string) corev1.NodeSelectorRequirement { + return corev1.NodeSelectorRequirement{Key: key, Operator: corev1.NodeSelectorOpIn, Values: []string{value}} + }) + deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ + NodeSelector: nodeSelector, + NodePreferences: requirements, + NodeRequirements: requirements, + }}) + Env.ExpectCreated(NodeClass, NodePool, deployment) + Env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) + Env.ExpectCreatedNodeCount("==", 1) + }) + It("should support well-known labels for an accelerator (inferentia2)", func() { + nodeSelector := map[string]string{ + v1.LabelInstanceAcceleratorName: "inferentia", + v1.LabelInstanceAcceleratorManufacturer: "aws", + v1.LabelInstanceAcceleratorCount: "1", + } + selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels + requirements := lo.MapToSlice(nodeSelector, func(key string, value string) corev1.NodeSelectorRequirement { + return corev1.NodeSelectorRequirement{Key: key, Operator: corev1.NodeSelectorOpIn, Values: []string{value}} + }) + deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ + NodeSelector: nodeSelector, + NodePreferences: requirements, + NodeRequirements: requirements, + }}) + Env.ExpectCreated(NodeClass, NodePool, deployment) + Env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) + Env.ExpectCreatedNodeCount("==", 1) + }) + // Windows tests are can flake due to the instance types that are used in testing. + // The VPC Resource controller will need to support the instance types that are used. + // If the instance type is not supported by the controller resource `vpc.amazonaws.com/PrivateIPv4Address` will not register. + // Issue: https://github.com/aws/karpenter-provider-aws/issues/4472 + // See: https://github.com/aws/amazon-vpc-resource-controller-k8s/blob/master/pkg/aws/vpc/limits.go + It("should support well-known labels for windows-build version", func() { + Env.ExpectWindowsIPAMEnabled() + DeferCleanup(func() { + Env.ExpectWindowsIPAMDisabled() + }) + + nodeSelector := map[string]string{ + // Well Known + corev1.LabelWindowsBuild: v1.Windows2022Build, + corev1.LabelOSStable: string(corev1.Windows), // Specify the OS to enable vpc-resource-controller to inject the PrivateIPv4Address resource + } + selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels + requirements := lo.MapToSlice(nodeSelector, func(key string, value string) corev1.NodeSelectorRequirement { + return corev1.NodeSelectorRequirement{Key: key, Operator: corev1.NodeSelectorOpIn, Values: []string{value}} + }) + deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ + NodeSelector: nodeSelector, + NodePreferences: requirements, + NodeRequirements: requirements, + Image: environmentaws.WindowsDefaultImage, + }}) + NodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "windows2022@latest"}} + test.ReplaceRequirements(NodePool, + karpv1.NodeSelectorRequirementWithMinValues{ + Key: corev1.LabelOSStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{string(corev1.Windows)}, + }, + ) + Env.ExpectCreated(NodeClass, NodePool, deployment) + Env.EventuallyExpectHealthyPodCountWithTimeout(time.Minute*15, labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) + Env.ExpectCreatedNodeCount("==", 1) + }) + DescribeTable("should support restricted label domain exceptions", func(domain string) { + // Assign labels to the nodepool so that it has known values + test.ReplaceRequirements(NodePool, + karpv1.NodeSelectorRequirementWithMinValues{Key: domain + "/team", Operator: corev1.NodeSelectorOpExists}, + karpv1.NodeSelectorRequirementWithMinValues{Key: domain + "/custom-label", Operator: corev1.NodeSelectorOpExists}, + karpv1.NodeSelectorRequirementWithMinValues{Key: "subdomain." + domain + "/custom-label", Operator: corev1.NodeSelectorOpExists}, + ) + nodeSelector := map[string]string{ + domain + "/team": "team-1", + domain + "/custom-label": "custom-value", + "subdomain." + domain + "/custom-label": "custom-value", + } + selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels + requirements := lo.MapToSlice(nodeSelector, func(key string, value string) corev1.NodeSelectorRequirement { + return corev1.NodeSelectorRequirement{Key: key, Operator: corev1.NodeSelectorOpIn, Values: []string{value}} + }) + deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ + NodeSelector: nodeSelector, + NodePreferences: requirements, + NodeRequirements: requirements, + }}) + Env.ExpectCreated(NodeClass, NodePool, deployment) + Env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) + node := Env.ExpectCreatedNodeCount("==", 1)[0] + // Ensure that the requirements/labels specified above are propagated onto the node + for k, v := range nodeSelector { + Expect(node.Labels).To(HaveKeyWithValue(k, v)) + } + }, + Entry("node-restriction.kuberentes.io", "node-restriction.kuberentes.io"), + Entry("node.kubernetes.io", "node.kubernetes.io"), + Entry("kops.k8s.io", "kops.k8s.io"), + ) + It("should support well-known labels for capacity tenancy", func() { + nodeSelector := map[string]string{ + v1.LabelInstanceTenancy: string(ec2types.CapacityReservationTenancyDedicated), + } + selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels + requirements := lo.MapToSlice(nodeSelector, func(key string, value string) corev1.NodeSelectorRequirement { + return corev1.NodeSelectorRequirement{Key: key, Operator: corev1.NodeSelectorOpIn, Values: []string{value}} + }) + deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ + NodeSelector: nodeSelector, + NodePreferences: requirements, + NodeRequirements: requirements, + }}) + Env.ExpectCreated(NodeClass, NodePool, deployment) + Env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) + Env.ExpectCreatedNodeCount("==", 1) + }) + }) + + Context("Provisioning", func() { + It("should provision a node for naked pods", func() { + pod := test.Pod() + + Env.ExpectCreated(NodeClass, NodePool, pod) + Env.EventuallyExpectHealthy(pod) + Env.ExpectCreatedNodeCount("==", 1) + }) + It("should honor minValuesPolicy when provisioning a node", func() { + eventClient := debug.NewEventClient(Env.Client) + pod := test.Pod() + nodePoolWithMinValues := test.ReplaceRequirements(NodePool, karpv1.NodeSelectorRequirementWithMinValues{ + Key: corev1.LabelInstanceTypeStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{"c5.large", "invalid-instance-type-1", "invalid-instance-type-2"}, + MinValues: lo.ToPtr(3), + }) + Env.ExpectCreated(NodeClass, nodePoolWithMinValues, pod) + + // minValues should only be relaxed when policy is set to BestEffort + if minValuesPolicy == options.MinValuesPolicyBestEffort { + Env.EventuallyExpectHealthy(pod) + Env.ExpectCreatedNodeCount("==", 1) + nodeClaim := Env.ExpectNodeClaimCount("==", 1) + Expect(nodeClaim[0].Annotations).To(HaveKeyWithValue(karpv1.NodeClaimMinValuesRelaxedAnnotationKey, "true")) + Expect(nodeClaim[0].Spec.Requirements).To(ContainElement(karpv1.NodeSelectorRequirementWithMinValues{ + Key: corev1.LabelInstanceTypeStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{"c5.large"}, + MinValues: lo.ToPtr(1), + })) + } else { + Env.ExpectExists(pod) + // Give a min for the scheduling decision to be done. + Env.ConsistentlyExpectPendingPods(time.Minute, pod) + Env.EventuallyExpectNodeCount("==", 0) + Env.ExpectNodeClaimCount("==", 0) + events, err := eventClient.GetEvents(Env.Context, "NodePool") + Expect(err).ToNot(HaveOccurred()) + key, found := lo.FindKeyBy(events, func(k corev1.ObjectReference, v *corev1.EventList) bool { + return k.Name == nodePoolWithMinValues.Name && + k.Namespace == nodePoolWithMinValues.Namespace + }) + Expect(found).To(BeTrue()) + _, found = lo.Find(events[key].Items, func(e corev1.Event) bool { + return e.InvolvedObject.Name == nodePoolWithMinValues.Name && + e.InvolvedObject.Namespace == nodePoolWithMinValues.Namespace && + e.Message == "NodePool requirements filtered out all compatible available instance types due to minValues incompatibility" + }) + Expect(found).To(BeTrue()) + } + }) + It("should provision a node for a deployment", Label(debug.NoWatch), Label(debug.NoEvents), func() { + deployment := test.Deployment(test.DeploymentOptions{Replicas: 50}) + Env.ExpectCreated(NodeClass, NodePool, deployment) + Env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) + Env.ExpectCreatedNodeCount("<=", 2) // should probably all land on a single node, but at worst two depending on batching + }) + It("should provision a node for a self-affinity deployment", func() { + // just two pods as they all need to land on the same node + podLabels := map[string]string{"test": "self-affinity"} + deployment := test.Deployment(test.DeploymentOptions{ + Replicas: 2, + PodOptions: test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Labels: podLabels, + }, + PodRequirements: []corev1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{MatchLabels: podLabels}, + TopologyKey: corev1.LabelHostname, + }, + }, + }, + }) + + Env.ExpectCreated(NodeClass, NodePool, deployment) + Env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), 2) + Env.ExpectCreatedNodeCount("==", 1) + }) + It("should provision three nodes for a zonal topology spread", func() { + // one pod per zone + podLabels := map[string]string{"test": "zonal-spread"} + deployment := test.Deployment(test.DeploymentOptions{ + Replicas: 3, + PodOptions: test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Labels: podLabels, + }, + TopologySpreadConstraints: []corev1.TopologySpreadConstraint{ + { + MaxSkew: 1, + TopologyKey: corev1.LabelTopologyZone, + WhenUnsatisfiable: corev1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: podLabels}, + MinDomains: lo.ToPtr(int32(3)), + }, + }, + }, + }) + + Env.ExpectCreated(NodeClass, NodePool, deployment) + Env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(podLabels), 3) + // Karpenter will launch three nodes, however if all three nodes don't get register with the cluster at the same time, two pods will be placed on one node. + // This can result in a case where all 3 pods are healthy, while there are only two created nodes. + // In that case, we still expect to eventually have three nodes. + Env.EventuallyExpectNodeCount("==", 3) + }) + It("should provision a node using a NodePool with higher priority", func() { + NodePoolLowPri := test.NodePool(karpv1.NodePool{ + Spec: karpv1.NodePoolSpec{ + Weight: lo.ToPtr(int32(10)), + Template: karpv1.NodeClaimTemplate{ + Spec: karpv1.NodeClaimTemplateSpec{ + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(NodeClass).Group, + Kind: object.GVK(NodeClass).Kind, + Name: NodeClass.Name, + }, + Requirements: []karpv1.NodeSelectorRequirementWithMinValues{ + { + Key: corev1.LabelOSStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{string(corev1.Linux)}, + }, + { + Key: corev1.LabelInstanceTypeStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{"t3.nano"}, + }, + }, + }, + }, + }, + }) + NodePoolHighPri := test.NodePool(karpv1.NodePool{ + Spec: karpv1.NodePoolSpec{ + Weight: lo.ToPtr(int32(100)), + Template: karpv1.NodeClaimTemplate{ + Spec: karpv1.NodeClaimTemplateSpec{ + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(NodeClass).Group, + Kind: object.GVK(NodeClass).Kind, + Name: NodeClass.Name, + }, + Requirements: []karpv1.NodeSelectorRequirementWithMinValues{ + { + Key: corev1.LabelOSStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{string(corev1.Linux)}, + }, + { + Key: corev1.LabelInstanceTypeStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{"c5.large"}, + }, + }, + }, + }, + }, + }) + pod := test.Pod() + Env.ExpectCreated(pod, NodeClass, NodePoolLowPri, NodePoolHighPri) + Env.EventuallyExpectHealthy(pod) + Env.ExpectCreatedNodeCount("==", 1) + Expect(Env.GetInstance(pod.Spec.NodeName).InstanceType).To(Equal(ec2types.InstanceType("c5.large"))) + Expect(Env.GetNode(pod.Spec.NodeName).Labels[karpv1.NodePoolLabelKey]).To(Equal(NodePoolHighPri.Name)) + }) + It("should provision a flex node for a pod", func() { + selectors.Insert(v1.LabelInstanceCapabilityFlex) + pod := test.Pod() + nodePoolWithMinValues := test.ReplaceRequirements(NodePool, karpv1.NodeSelectorRequirementWithMinValues{ + Key: v1.LabelInstanceCapabilityFlex, + Operator: corev1.NodeSelectorOpIn, + Values: []string{"true"}, + }) + Env.ExpectCreated(NodeClass, nodePoolWithMinValues, pod) + Env.EventuallyExpectHealthy(pod) + Env.ExpectCreatedNodeCount("==", 1) + Expect(Env.GetNode(pod.Spec.NodeName).Labels).To(And(HaveKeyWithValue(corev1.LabelInstanceType, ContainSubstring("flex")))) + }) + + DescribeTable( + "should provision a right-sized node when a pod has InitContainers (cpu)", + func(expectedNodeCPU string, containerRequirements corev1.ResourceRequirements, initContainers ...corev1.Container) { + if Env.K8sMinorVersion() < 29 { + Skip("native sidecar containers are only enabled on EKS 1.29+") + } + + labels := map[string]string{"test": test.RandomName()} + // Create a buffer pod to even out the total resource requests regardless of the daemonsets on the cluster. Assumes + // CPU is the resource in contention and that total daemonset CPU requests <= 3. + dsBufferPod := test.Pod(test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + PodRequirements: []corev1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + TopologyKey: corev1.LabelHostname, + }}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: func() resource.Quantity { + dsOverhead := Env.GetDaemonSetOverhead(NodePool) + base := lo.ToPtr(resource.MustParse("3")) + base.Sub(*dsOverhead.Cpu()) + return *base + }(), + }, + }, + }) + + test.ReplaceRequirements(NodePool, karpv1.NodeSelectorRequirementWithMinValues{ + Key: v1.LabelInstanceCPU, + Operator: corev1.NodeSelectorOpIn, + Values: []string{"4", "8"}, + }, karpv1.NodeSelectorRequirementWithMinValues{ + Key: v1.LabelInstanceCategory, + Operator: corev1.NodeSelectorOpNotIn, + Values: []string{"t"}, + }) + pod := test.Pod(test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + PodRequirements: []corev1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + TopologyKey: corev1.LabelHostname, + }}, + InitContainers: initContainers, + ResourceRequirements: containerRequirements, + }) + Env.ExpectCreated(NodePool, NodeClass, dsBufferPod, pod) + Env.EventuallyExpectHealthy(pod) + node := Env.ExpectCreatedNodeCount("==", 1)[0] + Expect(node.ObjectMeta.GetLabels()[v1.LabelInstanceCPU]).To(Equal(expectedNodeCPU)) + }, + Entry("sidecar requirements + later init requirements do exceed container requirements", "8", corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("400m")}, + }, ephemeralInitContainer(corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("300m")}, + }), corev1.Container{ + RestartPolicy: lo.ToPtr(corev1.ContainerRestartPolicyAlways), + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("350m")}, + }, + }, ephemeralInitContainer(corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, + })), + Entry("sidecar requirements + later init requirements do not exceed container requirements", "4", corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("400m")}, + }, ephemeralInitContainer(corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("300m")}, + }), corev1.Container{ + RestartPolicy: lo.ToPtr(corev1.ContainerRestartPolicyAlways), + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("350m")}, + }, + }, ephemeralInitContainer(corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("300m")}, + })), + Entry("init container requirements exceed all later requests", "8", corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("400m")}, + }, corev1.Container{ + RestartPolicy: lo.ToPtr(corev1.ContainerRestartPolicyAlways), + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m")}, + }, + }, ephemeralInitContainer(corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1500m")}, + }), corev1.Container{ + RestartPolicy: lo.ToPtr(corev1.ContainerRestartPolicyAlways), + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m")}, + }, + }), + ) + It("should provision a right-sized node when a pod has InitContainers (mixed resources)", func() { + if Env.K8sMinorVersion() < 29 { + Skip("native sidecar containers are only enabled on EKS 1.29+") + } + test.ReplaceRequirements(NodePool, karpv1.NodeSelectorRequirementWithMinValues{ + Key: v1.LabelInstanceCategory, + Operator: corev1.NodeSelectorOpNotIn, + Values: []string{"t"}, + }) + pod := test.Pod(test.PodOptions{ + InitContainers: []corev1.Container{ + { + RestartPolicy: lo.ToPtr(corev1.ContainerRestartPolicyAlways), + Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("128Mi"), + }}, + }, + ephemeralInitContainer(corev1.ResourceRequirements{Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("50m"), + corev1.ResourceMemory: resource.MustParse("4Gi"), + }}), + }, + ResourceRequirements: corev1.ResourceRequirements{Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("128Mi"), + }}, + }) + Env.ExpectCreated(NodePool, NodeClass, pod) + Env.EventuallyExpectHealthy(pod) + }) + + It("should provision a node for a pod with overlapping zone and zone-id requirements", func() { + subnetInfo := lo.UniqBy(Env.GetSubnetInfo(map[string]string{"karpenter.sh/discovery": Env.ClusterName}), func(s environmentaws.SubnetInfo) string { + return s.Zone + }) + Expect(len(subnetInfo)).To(BeNumerically(">=", 3)) + + // Create a pod with 'overlapping' zone and zone-id requirements. With two options for each label, but only one pair of zone-zoneID that maps to the + // same AZ, we will always expect the pod to be scheduled to that AZ. In this case, this is the mapping at zone[1]. + pod := test.Pod(test.PodOptions{ + NodeRequirements: []corev1.NodeSelectorRequirement{ + { + Key: corev1.LabelTopologyZone, + Operator: corev1.NodeSelectorOpIn, + Values: lo.Map(subnetInfo[0:2], func(info environmentaws.SubnetInfo, _ int) string { return info.Zone }), + }, + { + Key: v1.LabelTopologyZoneID, + Operator: corev1.NodeSelectorOpIn, + Values: lo.Map(subnetInfo[1:3], func(info environmentaws.SubnetInfo, _ int) string { return info.ZoneID }), + }, + }, + }) + Env.ExpectCreated(NodePool, NodeClass, pod) + node := Env.EventuallyExpectInitializedNodeCount("==", 1)[0] + Expect(node.Labels[corev1.LabelTopologyZone]).To(Equal(subnetInfo[1].Zone)) + Expect(node.Labels[v1.LabelTopologyZoneID]).To(Equal(subnetInfo[1].ZoneID)) + }) + It("should provision nodes for pods with zone-id requirements in the correct zone", func() { + // Each pod specifies a requirement on this expected zone, where the value is the matching zone for the + // required zone-id. This allows us to verify that Karpenter launched the node in the correct zone, even if + // it doesn't add the zone-id label and the label is added by CCM. If we didn't take this approach, we would + // succeed even if Karpenter doesn't add the label and /or incorrectly generated offerings on k8s 1.30 and + // above. This is an unlikely scenario, and adding this check is a defense in depth measure. + const expectedZoneLabel = "expected-zone-label" + test.ReplaceRequirements(NodePool, karpv1.NodeSelectorRequirementWithMinValues{ + Key: expectedZoneLabel, + Operator: corev1.NodeSelectorOpExists, + }) + + subnetInfo := lo.UniqBy(Env.GetSubnetInfo(map[string]string{"karpenter.sh/discovery": Env.ClusterName}), func(s environmentaws.SubnetInfo) string { + return s.Zone + }) + pods := lo.Map(subnetInfo, func(info environmentaws.SubnetInfo, _ int) *corev1.Pod { + return test.Pod(test.PodOptions{ + NodeRequirements: []corev1.NodeSelectorRequirement{ + { + Key: expectedZoneLabel, + Operator: corev1.NodeSelectorOpIn, + Values: []string{info.Zone}, + }, + { + Key: v1.LabelTopologyZoneID, + Operator: corev1.NodeSelectorOpIn, + Values: []string{info.ZoneID}, + }, + }, + }) + }) + + Env.ExpectCreated(NodePool, NodeClass) + for _, pod := range pods { + Env.ExpectCreated(pod) + } + nodes := Env.EventuallyExpectInitializedNodeCount("==", len(subnetInfo)) + for _, node := range nodes { + expectedZone, ok := node.Labels[expectedZoneLabel] + Expect(ok).To(BeTrue()) + Expect(node.Labels[corev1.LabelTopologyZone]).To(Equal(expectedZone)) + zoneInfo, ok := lo.Find(subnetInfo, func(info environmentaws.SubnetInfo) bool { + return info.Zone == expectedZone + }) + Expect(ok).To(BeTrue()) + Expect(node.Labels[v1.LabelTopologyZoneID]).To(Equal(zoneInfo.ZoneID)) + } + }) + }) + + Context("Capacity Reservations", func() { + var largeCapacityReservationID, xlargeCapacityReservationID string + BeforeAll(func() { + largeCapacityReservationID = environmentaws.ExpectCapacityReservationCreated( + Env.Context, + Env.EC2API, + ec2types.InstanceTypeM5Large, + Env.ZoneInfo[0].Zone, + 1, + nil, + nil, + ) + xlargeCapacityReservationID = environmentaws.ExpectCapacityReservationCreated( + Env.Context, + Env.EC2API, + ec2types.InstanceTypeM5Xlarge, + Env.ZoneInfo[0].Zone, + 2, + nil, + nil, + ) + }) + AfterAll(func() { + environmentaws.ExpectCapacityReservationsCanceled(Env.Context, Env.EC2API, largeCapacityReservationID, xlargeCapacityReservationID) + }) + BeforeEach(func() { + NodeClass.Spec.CapacityReservationSelectorTerms = []v1.CapacityReservationSelectorTerm{ + { + ID: largeCapacityReservationID, + }, + { + ID: xlargeCapacityReservationID, + }, + } + NodePool.Spec.Template.Spec.Requirements = []karpv1.NodeSelectorRequirementWithMinValues{ + { + Key: karpv1.CapacityTypeLabelKey, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.CapacityTypeOnDemand, karpv1.CapacityTypeReserved}, + }, + // We need to specify the OS label to prevent a daemonset with a Windows specific resource from scheduling against + // the node. Omitting this requirement will result in scheduling failures. + { + Key: corev1.LabelOSStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{string(corev1.Linux)}, + }, + } + }) + It("should schedule against a specific reservation ID", func() { + selectors.Insert(v1.LabelCapacityReservationID) + pod := test.Pod(test.PodOptions{ + NodeRequirements: []corev1.NodeSelectorRequirement{{ + Key: v1.LabelCapacityReservationID, + Operator: corev1.NodeSelectorOpIn, + Values: []string{xlargeCapacityReservationID}, + }}, + }) + Env.ExpectCreated(NodePool, NodeClass, pod) + + nc := Env.EventuallyExpectLaunchedNodeClaimCount("==", 1)[0] + req, ok := lo.Find(nc.Spec.Requirements, func(req karpv1.NodeSelectorRequirementWithMinValues) bool { + return req.Key == v1.LabelCapacityReservationID + }) + Expect(ok).To(BeTrue()) + Expect(req.Values).To(ConsistOf(xlargeCapacityReservationID)) + + Env.EventuallyExpectNodeClaimsReady(nc) + n := Env.EventuallyExpectNodeCount("==", 1)[0] + Expect(n.Labels).To(HaveKeyWithValue(karpv1.CapacityTypeLabelKey, karpv1.CapacityTypeReserved)) + Expect(n.Labels).To(HaveKeyWithValue(v1.LabelCapacityReservationType, string(v1.CapacityReservationTypeDefault))) + Expect(n.Labels).To(HaveKeyWithValue(v1.LabelCapacityReservationID, xlargeCapacityReservationID)) + }) + // NOTE: We're not exercising capacity blocks because it isn't possible to provision them ad-hoc for the use in an + // integration test. + It("should schedule against a specific reservation type", func() { + selectors.Insert(v1.LabelCapacityReservationType) + pod := test.Pod(test.PodOptions{ + NodeRequirements: []corev1.NodeSelectorRequirement{ + { + Key: v1.LabelCapacityReservationType, + Operator: corev1.NodeSelectorOpIn, + Values: []string{string(v1.CapacityReservationTypeDefault)}, + }, + // NOTE: Continue to select the xlarge instance to ensure we can use the large instance for the fallback test. ODCR + // capacity eventual consistency is inconsistent between different services (e.g. DescribeCapacityReservations and + // RunInstances) so we've allocated enough to ensure that each test can make use of them without overlapping. + { + Key: corev1.LabelInstanceTypeStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{string(ec2types.InstanceTypeM5Xlarge)}, + }, + }, + }) + Env.ExpectCreated(NodePool, NodeClass, pod) + + nc := Env.EventuallyExpectLaunchedNodeClaimCount("==", 1)[0] + req, ok := lo.Find(nc.Spec.Requirements, func(req karpv1.NodeSelectorRequirementWithMinValues) bool { + return req.Key == v1.LabelCapacityReservationType + }) + Expect(ok).To(BeTrue()) + Expect(req.Values).To(ConsistOf(string(v1.CapacityReservationTypeDefault))) + + Env.EventuallyExpectNodeClaimsReady(nc) + n := Env.EventuallyExpectNodeCount("==", 1)[0] + Expect(n.Labels).To(HaveKeyWithValue(karpv1.CapacityTypeLabelKey, karpv1.CapacityTypeReserved)) + Expect(n.Labels).To(HaveKeyWithValue(v1.LabelCapacityReservationType, string(v1.CapacityReservationTypeDefault))) + Expect(n.Labels).To(HaveKeyWithValue(v1.LabelCapacityReservationID, xlargeCapacityReservationID)) + }) + It("should fall back when compatible capacity reservations are exhausted", func() { + // We create two pods with self anti-affinity and a node selector on a specific instance type. The anti-affinity term + // ensures that we must provision 2 nodes, and the node selector selects upon an instance type with a single reserved + // instance available. As such, we should create a reserved NodeClaim for one pod, and an on-demand NodeClaim for the + // other. + podLabels := map[string]string{"foo": "bar"} + pods := test.Pods(2, test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Labels: podLabels, + }, + NodeRequirements: []corev1.NodeSelectorRequirement{{ + Key: corev1.LabelInstanceTypeStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{string(ec2types.InstanceTypeM5Large)}, + }}, + PodAntiRequirements: []corev1.PodAffinityTerm{{ + TopologyKey: corev1.LabelHostname, + LabelSelector: &metav1.LabelSelector{ + MatchLabels: podLabels, + }, + }}, + }) + Env.ExpectCreated(NodePool, NodeClass, pods[0], pods[1]) + + reservedCount := 0 + for _, nc := range Env.EventuallyExpectLaunchedNodeClaimCount("==", 2) { + req, ok := lo.Find(nc.Spec.Requirements, func(req karpv1.NodeSelectorRequirementWithMinValues) bool { + return req.Key == v1.LabelCapacityReservationID + }) + if ok { + reservedCount += 1 + Expect(req.Values).To(ConsistOf(largeCapacityReservationID)) + } + } + Expect(reservedCount).To(Equal(1)) + Env.EventuallyExpectNodeCount("==", 2) + }) + }) + + Context("Interruptible Capacity Resverations", func() { + var sourceReservationID, interruptibleReservationID, xlargeReservationID string + BeforeAll(func() { + sourceReservationID, interruptibleReservationID = environmentaws.ExpectInterruptibleCapacityReservationCreated( + Env.Context, + Env.EC2API, + ec2types.InstanceTypeM5Large, + Env.ZoneInfo[0].Zone, + 2, + 1, + nil, + ) + xlargeReservationID = environmentaws.ExpectCapacityReservationCreated( + Env.Context, + Env.EC2API, + ec2types.InstanceTypeM5Xlarge, + Env.ZoneInfo[0].Zone, + 1, + nil, + nil, + ) + }) + AfterAll(func() { + environmentaws.ExpectInterruptibleAndSourceCapacityCanceled(Env.Context, Env.EC2API, sourceReservationID, interruptibleReservationID) + environmentaws.ExpectCapacityReservationsCanceled(Env.Context, Env.EC2API, xlargeReservationID) + }) + BeforeEach(func() { + NodeClass.Spec.CapacityReservationSelectorTerms = []v1.CapacityReservationSelectorTerm{ + {ID: sourceReservationID}, {ID: interruptibleReservationID}, + } + NodePool.Spec.Template.Spec.Requirements = []karpv1.NodeSelectorRequirementWithMinValues{ + { + Key: karpv1.CapacityTypeLabelKey, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.CapacityTypeOnDemand, karpv1.CapacityTypeReserved}, + }, + { + Key: corev1.LabelOSStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{string(corev1.Linux)}, + }, + } + }) + + DescribeTable("should schedule against a specific reservation interruptibiltiy", func(interruptible bool) { + selectors.Insert(v1.LabelCapacityReservationInterruptible) + pod := test.Pod(test.PodOptions{ + NodeRequirements: []corev1.NodeSelectorRequirement{{ + Key: v1.LabelCapacityReservationInterruptible, + Operator: corev1.NodeSelectorOpIn, + Values: []string{strconv.FormatBool(interruptible)}, + }}, + }) + Env.ExpectCreated(NodePool, NodeClass, pod) + + nc := Env.EventuallyExpectLaunchedNodeClaimCount("==", 1)[0] + resReq, ok := lo.Find(nc.Spec.Requirements, func(req karpv1.NodeSelectorRequirementWithMinValues) bool { + return req.Key == v1.LabelCapacityReservationID + }) + Expect(ok).To(BeTrue()) + iReq, ok := lo.Find(nc.Spec.Requirements, func(req karpv1.NodeSelectorRequirementWithMinValues) bool { + return req.Key == v1.LabelCapacityReservationInterruptible + }) + Expect(ok).To(BeTrue()) + Expect(resReq.Values).To(ConsistOf(lo.Ternary(interruptible, interruptibleReservationID, sourceReservationID))) + Expect(iReq.Values).To(ConsistOf(strconv.FormatBool(interruptible))) + + Env.EventuallyExpectNodeClaimsReady(nc) + n := Env.EventuallyExpectNodeCount("==", 1)[0] + Expect(n.Labels).To(HaveKeyWithValue(v1.LabelCapacityReservationInterruptible, strconv.FormatBool(interruptible))) + }, + Entry("interruptible", true), + Entry("non-interruptible", false), + ) + It("should prioritize ODCR over IODCR if price is equal", func() { + pod := test.Pod(test.PodOptions{ + NodeRequirements: []corev1.NodeSelectorRequirement{{ + Key: corev1.LabelInstanceTypeStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{string(ec2types.InstanceTypeM5Large)}, + }}, + }) + Env.ExpectCreated(NodePool, NodeClass, pod) + + nc := Env.EventuallyExpectLaunchedNodeClaimCount("==", 1)[0] + req, ok := lo.Find(nc.Spec.Requirements, func(req karpv1.NodeSelectorRequirementWithMinValues) bool { + return req.Key == v1.LabelCapacityReservationID + }) + Expect(ok).To(BeTrue()) + // NodeClaim should have both reservations (but launch with ODCR) + Expect(req.Values).To(ConsistOf(sourceReservationID, interruptibleReservationID)) + + Env.EventuallyExpectNodeClaimsReady(nc) + n := Env.EventuallyExpectNodeCount("==", 1)[0] + Expect(n.Labels).To(HaveKeyWithValue(v1.LabelCapacityReservationInterruptible, "false")) + Expect(n.Labels).To(HaveKeyWithValue(v1.LabelCapacityReservationID, sourceReservationID)) + }) + It("should prioritize reservation with lower price", func() { + Env.ExpectCreated(NodeClass) + NodeClass.Spec.CapacityReservationSelectorTerms = []v1.CapacityReservationSelectorTerm{ + {ID: xlargeReservationID}, {ID: interruptibleReservationID}, + } + Env.ExpectUpdated(NodeClass) + + pod := test.Pod(test.PodOptions{ + NodeRequirements: []corev1.NodeSelectorRequirement{{ + Key: corev1.LabelInstanceTypeStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{string(ec2types.InstanceTypeM5Large), string(ec2types.InstanceTypeM5Xlarge)}, + }}, + }) + Env.ExpectCreated(NodePool, pod) + + nc := Env.EventuallyExpectLaunchedNodeClaimCount("==", 1)[0] + req, ok := lo.Find(nc.Spec.Requirements, func(req karpv1.NodeSelectorRequirementWithMinValues) bool { + return req.Key == v1.LabelCapacityReservationID + }) + Expect(ok).To(BeTrue()) + // NodeClaim should have both reservations (but launch in IODCR as its cheaper) + Expect(req.Values).To(ConsistOf(xlargeReservationID, interruptibleReservationID)) + + Env.EventuallyExpectNodeClaimsReady(nc) + n := Env.EventuallyExpectNodeCount("==", 1)[0] + Expect(n.Labels).To(HaveKeyWithValue(corev1.LabelInstanceTypeStable, string(ec2types.InstanceTypeM5Large))) + Expect(n.Labels).To(HaveKeyWithValue(v1.LabelCapacityReservationInterruptible, "true")) + }) + }) + It("should provision EFAs according to NodeClass configuration", func() { + pod := test.Pod(test.PodOptions{ + NodeRequirements: []corev1.NodeSelectorRequirement{ + {Key: v1.LabelInstanceCategory, Operator: corev1.NodeSelectorOpIn, Values: []string{"m"}}, + }, + }) + NodeClass.Spec.NetworkInterfaces = []*v1.NetworkInterface{ + {NetworkCardIndex: 0, DeviceIndex: 0, InterfaceType: v1.InterfaceTypeInterface}, + {NetworkCardIndex: 0, DeviceIndex: 1, InterfaceType: v1.InterfaceTypeEFAOnly}, + } + Env.ExpectCreated(NodeClass, NodePool, pod) + Env.EventuallyExpectHealthy(pod) + node := Env.ExpectCreatedNodeCount("==", 1)[0] + + instance := Env.GetInstance(node.Name) + networkInterfaces := instance.NetworkInterfaces + Expect(networkInterfaces).To(HaveLen(2)) + + for _, deviceIndex := range []int32{0, 1} { + networkInterface, found := lo.Find(networkInterfaces, func(i ec2types.InstanceNetworkInterface) bool { + Expect(i.Attachment).ToNot(BeNil()) + Expect(i.Attachment.DeviceIndex).ToNot(BeNil()) + return deviceIndex == lo.FromPtr(i.Attachment.DeviceIndex) + }) + Expect(found).To(Equal(true)) + + Expect(lo.FromPtr(networkInterface.InterfaceType)).To(Equal( + lo.Ternary(deviceIndex == 0, string(ec2types.NetworkInterfaceTypeInterface), string(ec2types.NetworkInterfaceTypeEfaOnly)), + )) + } + }) + }) +} + +func RegisterNodeOverlayTests() bool { + return Describe("Node Overlay", func() { + It("should provision the instance that is the cheepest based on a price adjustment node overlay applied", func() { + overlaiedInstanceType := "m7a.8xlarge" + pod := test.Pod() + nodeOverlay := test.NodeOverlay(v1alpha1.NodeOverlay{ + Spec: v1alpha1.NodeOverlaySpec{ + PriceAdjustment: lo.ToPtr("-99.99999999999%"), + Requirements: []v1alpha1.NodeSelectorRequirement{ + { + Key: corev1.LabelInstanceTypeStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{overlaiedInstanceType}, + }, + }, + }, + }) + Env.ExpectCreated(NodePool, NodeClass, nodeOverlay, pod) + Env.EventuallyExpectHealthy(pod) + node := Env.EventuallyExpectInitializedNodeCount("==", 1) + + instanceType, foundInstanceType := node[0].Labels[corev1.LabelInstanceTypeStable] + Expect(foundInstanceType).To(BeTrue()) + Expect(instanceType).To(Equal(overlaiedInstanceType)) + }) + It("should provision the instance that is the cheepest based on a price override node overlay applied", func() { + overlaiedInstanceType := "c7a.8xlarge" + pod := test.Pod() + nodeOverlay := test.NodeOverlay(v1alpha1.NodeOverlay{ + Spec: v1alpha1.NodeOverlaySpec{ + Price: lo.ToPtr("0.0000000232"), + Requirements: []v1alpha1.NodeSelectorRequirement{ + { + Key: corev1.LabelInstanceTypeStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{overlaiedInstanceType}, + }, + }, + }, + }) + Env.ExpectCreated(NodePool, NodeClass, nodeOverlay, pod) + Env.EventuallyExpectHealthy(pod) + node := Env.EventuallyExpectInitializedNodeCount("==", 1) + + instanceType, foundInstanceType := node[0].Labels[corev1.LabelInstanceTypeStable] + Expect(foundInstanceType).To(BeTrue()) + Expect(instanceType).To(Equal(overlaiedInstanceType)) + }) + It("should provision a node that matches hugepages resource requests", func() { + overlaiedInstanceType := "c7a.2xlarge" + pod := test.Pod(test.PodOptions{ + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: test.RandomCPU(), + corev1.ResourceMemory: test.RandomMemory(), + corev1.ResourceName("hugepages-2Mi"): resource.MustParse("100Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceName("hugepages-2Mi"): resource.MustParse("100Mi"), + }, + }, + }) + nodeOverlay := test.NodeOverlay(v1alpha1.NodeOverlay{ + Spec: v1alpha1.NodeOverlaySpec{ + Requirements: []v1alpha1.NodeSelectorRequirement{ + { + Key: corev1.LabelInstanceTypeStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{overlaiedInstanceType}, + }, + }, + Capacity: corev1.ResourceList{ + corev1.ResourceName("hugepages-2Mi"): resource.MustParse("4Gi"), + }, + }, + }) + + content, err := os.ReadFile("testdata/hugepage_userdata_input.sh") + Expect(err).To(BeNil()) + NodeClass.Spec.UserData = lo.ToPtr(string(content)) + + Env.ExpectCreated(NodePool, NodeClass, nodeOverlay, pod) + Env.EventuallyExpectHealthy(pod) + node := Env.EventuallyExpectInitializedNodeCount("==", 1) + + instanceType, foundInstanceType := node[0].Labels[corev1.LabelInstanceTypeStable] + Expect(foundInstanceType).To(BeTrue()) + Expect(instanceType).To(Equal(overlaiedInstanceType)) + }) + }) +} +func ephemeralInitContainer(requirements corev1.ResourceRequirements) corev1.Container { + return corev1.Container{ + Image: environmentaws.EphemeralInitContainerImage, + Command: []string{"/bin/sh"}, + Args: []string{"-c", "sleep 5"}, + Resources: requirements, + } +} diff --git a/test/suites/scheduling/suite_test.go b/test/suites/scheduling/suite_test.go deleted file mode 100644 index 6b6c077f9de2..000000000000 --- a/test/suites/scheduling/suite_test.go +++ /dev/null @@ -1,1171 +0,0 @@ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package scheduling_test - -import ( - "fmt" - "os" - "strconv" - "testing" - "time" - - ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" - - "github.com/awslabs/operatorpkg/object" - "github.com/samber/lo" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/util/sets" - - karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" - "sigs.k8s.io/karpenter/pkg/apis/v1alpha1" - "sigs.k8s.io/karpenter/pkg/operator/options" - "sigs.k8s.io/karpenter/pkg/test" - - v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" - "github.com/aws/karpenter-provider-aws/test/pkg/debug" - environmentaws "github.com/aws/karpenter-provider-aws/test/pkg/environment/aws" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var env *environmentaws.Environment -var nodeClass *v1.EC2NodeClass -var nodePool *karpv1.NodePool - -func TestScheduling(t *testing.T) { - RegisterFailHandler(Fail) - BeforeSuite(func() { - env = environmentaws.NewEnvironment(t) - }) - AfterSuite(func() { - env.Stop() - }) - RunSpecs(t, "Scheduling") -} - -var _ = BeforeEach(func() { - env.BeforeEach() - nodeClass = env.DefaultEC2NodeClass() - nodePool = env.DefaultNodePool(nodeClass) -}) -var _ = AfterEach(func() { env.Cleanup() }) -var _ = AfterEach(func() { env.AfterEach() }) -var _ = DescribeTableSubtree("Scheduling", Ordered, ContinueOnFailure, func(minValuesPolicy options.MinValuesPolicy) { - var selectors sets.Set[string] - - BeforeEach(func() { - // Make the NodePool requirements fully flexible, so we can match well-known label keys - nodePool = test.ReplaceRequirements(nodePool, - karpv1.NodeSelectorRequirementWithMinValues{ - Key: v1.LabelInstanceCategory, - Operator: corev1.NodeSelectorOpExists, - }, - karpv1.NodeSelectorRequirementWithMinValues{ - Key: v1.LabelInstanceGeneration, - Operator: corev1.NodeSelectorOpExists, - }, - ) - env.ExpectSettingsOverridden(corev1.EnvVar{Name: "MIN_VALUES_POLICY", Value: string(minValuesPolicy)}) - }) - BeforeAll(func() { - selectors = sets.New[string]() - }) - AfterAll(func() { - // Ensure that we're exercising all well known labels - Expect(lo.Keys(selectors)).To(ContainElements(append(karpv1.WellKnownLabels.UnsortedList(), lo.Keys(karpv1.NormalizedLabels)...))) - }) - - It("should apply annotations to the node", func() { - nodePool.Spec.Template.Annotations = map[string]string{ - "foo": "bar", - karpv1.DoNotDisruptAnnotationKey: "true", - } - pod := test.Pod() - env.ExpectCreated(nodeClass, nodePool, pod) - env.EventuallyExpectHealthy(pod) - env.ExpectCreatedNodeCount("==", 1) - Expect(env.GetNode(pod.Spec.NodeName).Annotations).To(And(HaveKeyWithValue("foo", "bar"), HaveKeyWithValue(karpv1.DoNotDisruptAnnotationKey, "true"))) - }) - - Context("Labels", func() { - It("should support well-known labels for instance type selection", func() { - nodeSelector := map[string]string{ - // Well Known - karpv1.NodePoolLabelKey: nodePool.Name, - corev1.LabelInstanceTypeStable: "c5.large", - // Well Known to AWS - v1.LabelInstanceHypervisor: "nitro", - v1.LabelInstanceCategory: "c", - v1.LabelInstanceGeneration: "5", - v1.LabelInstanceFamily: "c5", - v1.LabelInstanceSize: "large", - v1.LabelInstanceCPU: "2", - v1.LabelInstanceCPUManufacturer: "intel", - v1.LabelInstanceCPUSustainedClockSpeedMhz: "3400", - v1.LabelInstanceMemory: "4096", - v1.LabelInstanceEBSBandwidth: "4750", - v1.LabelInstanceNetworkBandwidth: "750", - } - selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels - requirements := lo.MapToSlice(nodeSelector, func(key string, value string) corev1.NodeSelectorRequirement { - return corev1.NodeSelectorRequirement{Key: key, Operator: corev1.NodeSelectorOpIn, Values: []string{value}} - }) - deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ - NodeSelector: nodeSelector, - NodePreferences: requirements, - NodeRequirements: requirements, - }}) - env.ExpectCreated(nodeClass, nodePool, deployment) - env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) - env.ExpectCreatedNodeCount("==", 1) - }) - It("should support well-known labels for zone id selection", func() { - selectors.Insert(v1.LabelTopologyZoneID) // Add node selector keys to selectors used in testing to ensure we test all labels - deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ - NodeRequirements: []corev1.NodeSelectorRequirement{ - { - Key: v1.LabelTopologyZoneID, - Operator: corev1.NodeSelectorOpIn, - Values: []string{env.GetSubnetInfo(map[string]string{"karpenter.sh/discovery": env.ClusterName})[0].ZoneID}, - }, - }, - }}) - env.ExpectCreated(nodeClass, nodePool, deployment) - env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) - env.ExpectCreatedNodeCount("==", 1) - }) - It("should support well-known labels for local NVME storage", func() { - selectors.Insert(v1.LabelInstanceLocalNVME) // Add node selector keys to selectors used in testing to ensure we test all labels - deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ - NodePreferences: []corev1.NodeSelectorRequirement{ - { - Key: v1.LabelInstanceLocalNVME, - Operator: corev1.NodeSelectorOpGt, - Values: []string{"0"}, - }, - }, - NodeRequirements: []corev1.NodeSelectorRequirement{ - { - Key: v1.LabelInstanceLocalNVME, - Operator: corev1.NodeSelectorOpGt, - Values: []string{"0"}, - }, - }, - }}) - env.ExpectCreated(nodeClass, nodePool, deployment) - env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) - env.ExpectCreatedNodeCount("==", 1) - }) - It("should support well-known labels for encryption in transit", func() { - selectors.Insert(v1.LabelInstanceEncryptionInTransitSupported) // Add node selector keys to selectors used in testing to ensure we test all labels - deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ - NodePreferences: []corev1.NodeSelectorRequirement{ - { - Key: v1.LabelInstanceEncryptionInTransitSupported, - Operator: corev1.NodeSelectorOpIn, - Values: []string{"true"}, - }, - }, - NodeRequirements: []corev1.NodeSelectorRequirement{ - { - Key: v1.LabelInstanceEncryptionInTransitSupported, - Operator: corev1.NodeSelectorOpIn, - Values: []string{"true"}, - }, - }, - }}) - env.ExpectCreated(nodeClass, nodePool, deployment) - env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) - env.ExpectCreatedNodeCount("==", 1) - }) - It("should support well-known deprecated labels", func() { - nodeSelector := map[string]string{ - // Deprecated Labels - corev1.LabelFailureDomainBetaRegion: env.Region, - corev1.LabelFailureDomainBetaZone: fmt.Sprintf("%sa", env.Region), - "topology.ebs.csi.aws.com/zone": fmt.Sprintf("%sa", env.Region), - - "beta.kubernetes.io/arch": "amd64", - "beta.kubernetes.io/os": "linux", - corev1.LabelInstanceType: "c5.large", - } - selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels - requirements := lo.MapToSlice(nodeSelector, func(key string, value string) corev1.NodeSelectorRequirement { - return corev1.NodeSelectorRequirement{Key: key, Operator: corev1.NodeSelectorOpIn, Values: []string{value}} - }) - deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ - NodeSelector: nodeSelector, - NodePreferences: requirements, - NodeRequirements: requirements, - }}) - env.ExpectCreated(nodeClass, nodePool, deployment) - env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) - env.ExpectCreatedNodeCount("==", 1) - }) - It("should support well-known labels for topology and architecture", func() { - nodeSelector := map[string]string{ - // Well Known - karpv1.NodePoolLabelKey: nodePool.Name, - corev1.LabelTopologyRegion: env.Region, - corev1.LabelTopologyZone: fmt.Sprintf("%sa", env.Region), - corev1.LabelOSStable: "linux", - corev1.LabelArchStable: "amd64", - karpv1.CapacityTypeLabelKey: karpv1.CapacityTypeOnDemand, - } - selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels - requirements := lo.MapToSlice(nodeSelector, func(key string, value string) corev1.NodeSelectorRequirement { - return corev1.NodeSelectorRequirement{Key: key, Operator: corev1.NodeSelectorOpIn, Values: []string{value}} - }) - deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ - NodeSelector: nodeSelector, - NodePreferences: requirements, - NodeRequirements: requirements, - }}) - env.ExpectCreated(nodeClass, nodePool, deployment) - env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) - env.ExpectCreatedNodeCount("==", 1) - }) - It("should support well-known labels for a gpu (nvidia)", func() { - nodeSelector := map[string]string{ - v1.LabelInstanceGPUName: "t4", - v1.LabelInstanceGPUMemory: "16384", - v1.LabelInstanceGPUManufacturer: "nvidia", - v1.LabelInstanceGPUCount: "1", - } - selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels - requirements := lo.MapToSlice(nodeSelector, func(key string, value string) corev1.NodeSelectorRequirement { - return corev1.NodeSelectorRequirement{Key: key, Operator: corev1.NodeSelectorOpIn, Values: []string{value}} - }) - deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ - NodeSelector: nodeSelector, - NodePreferences: requirements, - NodeRequirements: requirements, - }}) - env.ExpectCreated(nodeClass, nodePool, deployment) - env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) - env.ExpectCreatedNodeCount("==", 1) - }) - It("should support well-known labels for an accelerator (inferentia2)", func() { - nodeSelector := map[string]string{ - v1.LabelInstanceAcceleratorName: "inferentia", - v1.LabelInstanceAcceleratorManufacturer: "aws", - v1.LabelInstanceAcceleratorCount: "1", - } - selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels - requirements := lo.MapToSlice(nodeSelector, func(key string, value string) corev1.NodeSelectorRequirement { - return corev1.NodeSelectorRequirement{Key: key, Operator: corev1.NodeSelectorOpIn, Values: []string{value}} - }) - deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ - NodeSelector: nodeSelector, - NodePreferences: requirements, - NodeRequirements: requirements, - }}) - env.ExpectCreated(nodeClass, nodePool, deployment) - env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) - env.ExpectCreatedNodeCount("==", 1) - }) - // Windows tests are can flake due to the instance types that are used in testing. - // The VPC Resource controller will need to support the instance types that are used. - // If the instance type is not supported by the controller resource `vpc.amazonaws.com/PrivateIPv4Address` will not register. - // Issue: https://github.com/aws/karpenter-provider-aws/issues/4472 - // See: https://github.com/aws/amazon-vpc-resource-controller-k8s/blob/master/pkg/aws/vpc/limits.go - It("should support well-known labels for windows-build version", func() { - env.ExpectWindowsIPAMEnabled() - DeferCleanup(func() { - env.ExpectWindowsIPAMDisabled() - }) - - nodeSelector := map[string]string{ - // Well Known - corev1.LabelWindowsBuild: v1.Windows2022Build, - corev1.LabelOSStable: string(corev1.Windows), // Specify the OS to enable vpc-resource-controller to inject the PrivateIPv4Address resource - } - selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels - requirements := lo.MapToSlice(nodeSelector, func(key string, value string) corev1.NodeSelectorRequirement { - return corev1.NodeSelectorRequirement{Key: key, Operator: corev1.NodeSelectorOpIn, Values: []string{value}} - }) - deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ - NodeSelector: nodeSelector, - NodePreferences: requirements, - NodeRequirements: requirements, - Image: environmentaws.WindowsDefaultImage, - }}) - nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "windows2022@latest"}} - test.ReplaceRequirements(nodePool, - karpv1.NodeSelectorRequirementWithMinValues{ - Key: corev1.LabelOSStable, - Operator: corev1.NodeSelectorOpIn, - Values: []string{string(corev1.Windows)}, - }, - ) - env.ExpectCreated(nodeClass, nodePool, deployment) - env.EventuallyExpectHealthyPodCountWithTimeout(time.Minute*15, labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) - env.ExpectCreatedNodeCount("==", 1) - }) - DescribeTable("should support restricted label domain exceptions", func(domain string) { - // Assign labels to the nodepool so that it has known values - test.ReplaceRequirements(nodePool, - karpv1.NodeSelectorRequirementWithMinValues{Key: domain + "/team", Operator: corev1.NodeSelectorOpExists}, - karpv1.NodeSelectorRequirementWithMinValues{Key: domain + "/custom-label", Operator: corev1.NodeSelectorOpExists}, - karpv1.NodeSelectorRequirementWithMinValues{Key: "subdomain." + domain + "/custom-label", Operator: corev1.NodeSelectorOpExists}, - ) - nodeSelector := map[string]string{ - domain + "/team": "team-1", - domain + "/custom-label": "custom-value", - "subdomain." + domain + "/custom-label": "custom-value", - } - selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels - requirements := lo.MapToSlice(nodeSelector, func(key string, value string) corev1.NodeSelectorRequirement { - return corev1.NodeSelectorRequirement{Key: key, Operator: corev1.NodeSelectorOpIn, Values: []string{value}} - }) - deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ - NodeSelector: nodeSelector, - NodePreferences: requirements, - NodeRequirements: requirements, - }}) - env.ExpectCreated(nodeClass, nodePool, deployment) - env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) - node := env.ExpectCreatedNodeCount("==", 1)[0] - // Ensure that the requirements/labels specified above are propagated onto the node - for k, v := range nodeSelector { - Expect(node.Labels).To(HaveKeyWithValue(k, v)) - } - }, - Entry("node-restriction.kuberentes.io", "node-restriction.kuberentes.io"), - Entry("node.kubernetes.io", "node.kubernetes.io"), - Entry("kops.k8s.io", "kops.k8s.io"), - ) - It("should support well-known labels for capacity tenancy", func() { - nodeSelector := map[string]string{ - v1.LabelInstanceTenancy: string(ec2types.CapacityReservationTenancyDedicated), - } - selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels - requirements := lo.MapToSlice(nodeSelector, func(key string, value string) corev1.NodeSelectorRequirement { - return corev1.NodeSelectorRequirement{Key: key, Operator: corev1.NodeSelectorOpIn, Values: []string{value}} - }) - deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ - NodeSelector: nodeSelector, - NodePreferences: requirements, - NodeRequirements: requirements, - }}) - env.ExpectCreated(nodeClass, nodePool, deployment) - env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) - env.ExpectCreatedNodeCount("==", 1) - }) - }) - - Context("Provisioning", func() { - It("should provision a node for naked pods", func() { - pod := test.Pod() - - env.ExpectCreated(nodeClass, nodePool, pod) - env.EventuallyExpectHealthy(pod) - env.ExpectCreatedNodeCount("==", 1) - }) - It("should honor minValuesPolicy when provisioning a node", func() { - eventClient := debug.NewEventClient(env.Client) - pod := test.Pod() - nodePoolWithMinValues := test.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ - Key: corev1.LabelInstanceTypeStable, - Operator: corev1.NodeSelectorOpIn, - Values: []string{"c5.large", "invalid-instance-type-1", "invalid-instance-type-2"}, - MinValues: lo.ToPtr(3), - }) - env.ExpectCreated(nodeClass, nodePoolWithMinValues, pod) - - // minValues should only be relaxed when policy is set to BestEffort - if minValuesPolicy == options.MinValuesPolicyBestEffort { - env.EventuallyExpectHealthy(pod) - env.ExpectCreatedNodeCount("==", 1) - nodeClaim := env.ExpectNodeClaimCount("==", 1) - Expect(nodeClaim[0].Annotations).To(HaveKeyWithValue(karpv1.NodeClaimMinValuesRelaxedAnnotationKey, "true")) - Expect(nodeClaim[0].Spec.Requirements).To(ContainElement(karpv1.NodeSelectorRequirementWithMinValues{ - Key: corev1.LabelInstanceTypeStable, - Operator: corev1.NodeSelectorOpIn, - Values: []string{"c5.large"}, - MinValues: lo.ToPtr(1), - })) - } else { - env.ExpectExists(pod) - // Give a min for the scheduling decision to be done. - env.ConsistentlyExpectPendingPods(time.Minute, pod) - env.EventuallyExpectNodeCount("==", 0) - env.ExpectNodeClaimCount("==", 0) - events, err := eventClient.GetEvents(env.Context, "NodePool") - Expect(err).ToNot(HaveOccurred()) - key, found := lo.FindKeyBy(events, func(k corev1.ObjectReference, v *corev1.EventList) bool { - return k.Name == nodePoolWithMinValues.Name && - k.Namespace == nodePoolWithMinValues.Namespace - }) - Expect(found).To(BeTrue()) - _, found = lo.Find(events[key].Items, func(e corev1.Event) bool { - return e.InvolvedObject.Name == nodePoolWithMinValues.Name && - e.InvolvedObject.Namespace == nodePoolWithMinValues.Namespace && - e.Message == "NodePool requirements filtered out all compatible available instance types due to minValues incompatibility" - }) - Expect(found).To(BeTrue()) - } - }) - It("should provision a node for a deployment", Label(debug.NoWatch), Label(debug.NoEvents), func() { - deployment := test.Deployment(test.DeploymentOptions{Replicas: 50}) - env.ExpectCreated(nodeClass, nodePool, deployment) - env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) - env.ExpectCreatedNodeCount("<=", 2) // should probably all land on a single node, but at worst two depending on batching - }) - It("should provision a node for a self-affinity deployment", func() { - // just two pods as they all need to land on the same node - podLabels := map[string]string{"test": "self-affinity"} - deployment := test.Deployment(test.DeploymentOptions{ - Replicas: 2, - PodOptions: test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{ - Labels: podLabels, - }, - PodRequirements: []corev1.PodAffinityTerm{ - { - LabelSelector: &metav1.LabelSelector{MatchLabels: podLabels}, - TopologyKey: corev1.LabelHostname, - }, - }, - }, - }) - - env.ExpectCreated(nodeClass, nodePool, deployment) - env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), 2) - env.ExpectCreatedNodeCount("==", 1) - }) - It("should provision three nodes for a zonal topology spread", func() { - // one pod per zone - podLabels := map[string]string{"test": "zonal-spread"} - deployment := test.Deployment(test.DeploymentOptions{ - Replicas: 3, - PodOptions: test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{ - Labels: podLabels, - }, - TopologySpreadConstraints: []corev1.TopologySpreadConstraint{ - { - MaxSkew: 1, - TopologyKey: corev1.LabelTopologyZone, - WhenUnsatisfiable: corev1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: podLabels}, - MinDomains: lo.ToPtr(int32(3)), - }, - }, - }, - }) - - env.ExpectCreated(nodeClass, nodePool, deployment) - env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(podLabels), 3) - // Karpenter will launch three nodes, however if all three nodes don't get register with the cluster at the same time, two pods will be placed on one node. - // This can result in a case where all 3 pods are healthy, while there are only two created nodes. - // In that case, we still expect to eventually have three nodes. - env.EventuallyExpectNodeCount("==", 3) - }) - It("should provision a node using a NodePool with higher priority", func() { - nodePoolLowPri := test.NodePool(karpv1.NodePool{ - Spec: karpv1.NodePoolSpec{ - Weight: lo.ToPtr(int32(10)), - Template: karpv1.NodeClaimTemplate{ - Spec: karpv1.NodeClaimTemplateSpec{ - NodeClassRef: &karpv1.NodeClassReference{ - Group: object.GVK(nodeClass).Group, - Kind: object.GVK(nodeClass).Kind, - Name: nodeClass.Name, - }, - Requirements: []karpv1.NodeSelectorRequirementWithMinValues{ - { - Key: corev1.LabelOSStable, - Operator: corev1.NodeSelectorOpIn, - Values: []string{string(corev1.Linux)}, - }, - { - Key: corev1.LabelInstanceTypeStable, - Operator: corev1.NodeSelectorOpIn, - Values: []string{"t3.nano"}, - }, - }, - }, - }, - }, - }) - nodePoolHighPri := test.NodePool(karpv1.NodePool{ - Spec: karpv1.NodePoolSpec{ - Weight: lo.ToPtr(int32(100)), - Template: karpv1.NodeClaimTemplate{ - Spec: karpv1.NodeClaimTemplateSpec{ - NodeClassRef: &karpv1.NodeClassReference{ - Group: object.GVK(nodeClass).Group, - Kind: object.GVK(nodeClass).Kind, - Name: nodeClass.Name, - }, - Requirements: []karpv1.NodeSelectorRequirementWithMinValues{ - { - Key: corev1.LabelOSStable, - Operator: corev1.NodeSelectorOpIn, - Values: []string{string(corev1.Linux)}, - }, - { - Key: corev1.LabelInstanceTypeStable, - Operator: corev1.NodeSelectorOpIn, - Values: []string{"c5.large"}, - }, - }, - }, - }, - }, - }) - pod := test.Pod() - env.ExpectCreated(pod, nodeClass, nodePoolLowPri, nodePoolHighPri) - env.EventuallyExpectHealthy(pod) - env.ExpectCreatedNodeCount("==", 1) - Expect(env.GetInstance(pod.Spec.NodeName).InstanceType).To(Equal(ec2types.InstanceType("c5.large"))) - Expect(env.GetNode(pod.Spec.NodeName).Labels[karpv1.NodePoolLabelKey]).To(Equal(nodePoolHighPri.Name)) - }) - It("should provision a flex node for a pod", func() { - selectors.Insert(v1.LabelInstanceCapabilityFlex) - pod := test.Pod() - nodePoolWithMinValues := test.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ - Key: v1.LabelInstanceCapabilityFlex, - Operator: corev1.NodeSelectorOpIn, - Values: []string{"true"}, - }) - env.ExpectCreated(nodeClass, nodePoolWithMinValues, pod) - env.EventuallyExpectHealthy(pod) - env.ExpectCreatedNodeCount("==", 1) - Expect(env.GetNode(pod.Spec.NodeName).Labels).To(And(HaveKeyWithValue(corev1.LabelInstanceType, ContainSubstring("flex")))) - }) - - DescribeTable( - "should provision a right-sized node when a pod has InitContainers (cpu)", - func(expectedNodeCPU string, containerRequirements corev1.ResourceRequirements, initContainers ...corev1.Container) { - if env.K8sMinorVersion() < 29 { - Skip("native sidecar containers are only enabled on EKS 1.29+") - } - - labels := map[string]string{"test": test.RandomName()} - // Create a buffer pod to even out the total resource requests regardless of the daemonsets on the cluster. Assumes - // CPU is the resource in contention and that total daemonset CPU requests <= 3. - dsBufferPod := test.Pod(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{ - Labels: labels, - }, - PodRequirements: []corev1.PodAffinityTerm{{ - LabelSelector: &metav1.LabelSelector{ - MatchLabels: labels, - }, - TopologyKey: corev1.LabelHostname, - }}, - ResourceRequirements: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceCPU: func() resource.Quantity { - dsOverhead := env.GetDaemonSetOverhead(nodePool) - base := lo.ToPtr(resource.MustParse("3")) - base.Sub(*dsOverhead.Cpu()) - return *base - }(), - }, - }, - }) - - test.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ - Key: v1.LabelInstanceCPU, - Operator: corev1.NodeSelectorOpIn, - Values: []string{"4", "8"}, - }, karpv1.NodeSelectorRequirementWithMinValues{ - Key: v1.LabelInstanceCategory, - Operator: corev1.NodeSelectorOpNotIn, - Values: []string{"t"}, - }) - pod := test.Pod(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{ - Labels: labels, - }, - PodRequirements: []corev1.PodAffinityTerm{{ - LabelSelector: &metav1.LabelSelector{ - MatchLabels: labels, - }, - TopologyKey: corev1.LabelHostname, - }}, - InitContainers: initContainers, - ResourceRequirements: containerRequirements, - }) - env.ExpectCreated(nodePool, nodeClass, dsBufferPod, pod) - env.EventuallyExpectHealthy(pod) - node := env.ExpectCreatedNodeCount("==", 1)[0] - Expect(node.ObjectMeta.GetLabels()[v1.LabelInstanceCPU]).To(Equal(expectedNodeCPU)) - }, - Entry("sidecar requirements + later init requirements do exceed container requirements", "8", corev1.ResourceRequirements{ - Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("400m")}, - }, ephemeralInitContainer(corev1.ResourceRequirements{ - Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("300m")}, - }), corev1.Container{ - RestartPolicy: lo.ToPtr(corev1.ContainerRestartPolicyAlways), - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("350m")}, - }, - }, ephemeralInitContainer(corev1.ResourceRequirements{ - Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, - })), - Entry("sidecar requirements + later init requirements do not exceed container requirements", "4", corev1.ResourceRequirements{ - Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("400m")}, - }, ephemeralInitContainer(corev1.ResourceRequirements{ - Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("300m")}, - }), corev1.Container{ - RestartPolicy: lo.ToPtr(corev1.ContainerRestartPolicyAlways), - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("350m")}, - }, - }, ephemeralInitContainer(corev1.ResourceRequirements{ - Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("300m")}, - })), - Entry("init container requirements exceed all later requests", "8", corev1.ResourceRequirements{ - Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("400m")}, - }, corev1.Container{ - RestartPolicy: lo.ToPtr(corev1.ContainerRestartPolicyAlways), - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m")}, - }, - }, ephemeralInitContainer(corev1.ResourceRequirements{ - Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1500m")}, - }), corev1.Container{ - RestartPolicy: lo.ToPtr(corev1.ContainerRestartPolicyAlways), - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m")}, - }, - }), - ) - It("should provision a right-sized node when a pod has InitContainers (mixed resources)", func() { - if env.K8sMinorVersion() < 29 { - Skip("native sidecar containers are only enabled on EKS 1.29+") - } - test.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ - Key: v1.LabelInstanceCategory, - Operator: corev1.NodeSelectorOpNotIn, - Values: []string{"t"}, - }) - pod := test.Pod(test.PodOptions{ - InitContainers: []corev1.Container{ - { - RestartPolicy: lo.ToPtr(corev1.ContainerRestartPolicyAlways), - Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("100m"), - corev1.ResourceMemory: resource.MustParse("128Mi"), - }}, - }, - ephemeralInitContainer(corev1.ResourceRequirements{Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("50m"), - corev1.ResourceMemory: resource.MustParse("4Gi"), - }}), - }, - ResourceRequirements: corev1.ResourceRequirements{Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("100m"), - corev1.ResourceMemory: resource.MustParse("128Mi"), - }}, - }) - env.ExpectCreated(nodePool, nodeClass, pod) - env.EventuallyExpectHealthy(pod) - }) - - It("should provision a node for a pod with overlapping zone and zone-id requirements", func() { - subnetInfo := lo.UniqBy(env.GetSubnetInfo(map[string]string{"karpenter.sh/discovery": env.ClusterName}), func(s environmentaws.SubnetInfo) string { - return s.Zone - }) - Expect(len(subnetInfo)).To(BeNumerically(">=", 3)) - - // Create a pod with 'overlapping' zone and zone-id requirements. With two options for each label, but only one pair of zone-zoneID that maps to the - // same AZ, we will always expect the pod to be scheduled to that AZ. In this case, this is the mapping at zone[1]. - pod := test.Pod(test.PodOptions{ - NodeRequirements: []corev1.NodeSelectorRequirement{ - { - Key: corev1.LabelTopologyZone, - Operator: corev1.NodeSelectorOpIn, - Values: lo.Map(subnetInfo[0:2], func(info environmentaws.SubnetInfo, _ int) string { return info.Zone }), - }, - { - Key: v1.LabelTopologyZoneID, - Operator: corev1.NodeSelectorOpIn, - Values: lo.Map(subnetInfo[1:3], func(info environmentaws.SubnetInfo, _ int) string { return info.ZoneID }), - }, - }, - }) - env.ExpectCreated(nodePool, nodeClass, pod) - node := env.EventuallyExpectInitializedNodeCount("==", 1)[0] - Expect(node.Labels[corev1.LabelTopologyZone]).To(Equal(subnetInfo[1].Zone)) - Expect(node.Labels[v1.LabelTopologyZoneID]).To(Equal(subnetInfo[1].ZoneID)) - }) - It("should provision nodes for pods with zone-id requirements in the correct zone", func() { - // Each pod specifies a requirement on this expected zone, where the value is the matching zone for the - // required zone-id. This allows us to verify that Karpenter launched the node in the correct zone, even if - // it doesn't add the zone-id label and the label is added by CCM. If we didn't take this approach, we would - // succeed even if Karpenter doesn't add the label and /or incorrectly generated offerings on k8s 1.30 and - // above. This is an unlikely scenario, and adding this check is a defense in depth measure. - const expectedZoneLabel = "expected-zone-label" - test.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ - Key: expectedZoneLabel, - Operator: corev1.NodeSelectorOpExists, - }) - - subnetInfo := lo.UniqBy(env.GetSubnetInfo(map[string]string{"karpenter.sh/discovery": env.ClusterName}), func(s environmentaws.SubnetInfo) string { - return s.Zone - }) - pods := lo.Map(subnetInfo, func(info environmentaws.SubnetInfo, _ int) *corev1.Pod { - return test.Pod(test.PodOptions{ - NodeRequirements: []corev1.NodeSelectorRequirement{ - { - Key: expectedZoneLabel, - Operator: corev1.NodeSelectorOpIn, - Values: []string{info.Zone}, - }, - { - Key: v1.LabelTopologyZoneID, - Operator: corev1.NodeSelectorOpIn, - Values: []string{info.ZoneID}, - }, - }, - }) - }) - - env.ExpectCreated(nodePool, nodeClass) - for _, pod := range pods { - env.ExpectCreated(pod) - } - nodes := env.EventuallyExpectInitializedNodeCount("==", len(subnetInfo)) - for _, node := range nodes { - expectedZone, ok := node.Labels[expectedZoneLabel] - Expect(ok).To(BeTrue()) - Expect(node.Labels[corev1.LabelTopologyZone]).To(Equal(expectedZone)) - zoneInfo, ok := lo.Find(subnetInfo, func(info environmentaws.SubnetInfo) bool { - return info.Zone == expectedZone - }) - Expect(ok).To(BeTrue()) - Expect(node.Labels[v1.LabelTopologyZoneID]).To(Equal(zoneInfo.ZoneID)) - } - }) - }) - - Context("Capacity Reservations", func() { - var largeCapacityReservationID, xlargeCapacityReservationID string - BeforeAll(func() { - largeCapacityReservationID = environmentaws.ExpectCapacityReservationCreated( - env.Context, - env.EC2API, - ec2types.InstanceTypeM5Large, - env.ZoneInfo[0].Zone, - 1, - nil, - nil, - ) - xlargeCapacityReservationID = environmentaws.ExpectCapacityReservationCreated( - env.Context, - env.EC2API, - ec2types.InstanceTypeM5Xlarge, - env.ZoneInfo[0].Zone, - 2, - nil, - nil, - ) - }) - AfterAll(func() { - environmentaws.ExpectCapacityReservationsCanceled(env.Context, env.EC2API, largeCapacityReservationID, xlargeCapacityReservationID) - }) - BeforeEach(func() { - nodeClass.Spec.CapacityReservationSelectorTerms = []v1.CapacityReservationSelectorTerm{ - { - ID: largeCapacityReservationID, - }, - { - ID: xlargeCapacityReservationID, - }, - } - nodePool.Spec.Template.Spec.Requirements = []karpv1.NodeSelectorRequirementWithMinValues{ - { - Key: karpv1.CapacityTypeLabelKey, - Operator: corev1.NodeSelectorOpIn, - Values: []string{karpv1.CapacityTypeOnDemand, karpv1.CapacityTypeReserved}, - }, - // We need to specify the OS label to prevent a daemonset with a Windows specific resource from scheduling against - // the node. Omitting this requirement will result in scheduling failures. - { - Key: corev1.LabelOSStable, - Operator: corev1.NodeSelectorOpIn, - Values: []string{string(corev1.Linux)}, - }, - } - }) - It("should schedule against a specific reservation ID", func() { - selectors.Insert(v1.LabelCapacityReservationID) - pod := test.Pod(test.PodOptions{ - NodeRequirements: []corev1.NodeSelectorRequirement{{ - Key: v1.LabelCapacityReservationID, - Operator: corev1.NodeSelectorOpIn, - Values: []string{xlargeCapacityReservationID}, - }}, - }) - env.ExpectCreated(nodePool, nodeClass, pod) - - nc := env.EventuallyExpectLaunchedNodeClaimCount("==", 1)[0] - req, ok := lo.Find(nc.Spec.Requirements, func(req karpv1.NodeSelectorRequirementWithMinValues) bool { - return req.Key == v1.LabelCapacityReservationID - }) - Expect(ok).To(BeTrue()) - Expect(req.Values).To(ConsistOf(xlargeCapacityReservationID)) - - env.EventuallyExpectNodeClaimsReady(nc) - n := env.EventuallyExpectNodeCount("==", 1)[0] - Expect(n.Labels).To(HaveKeyWithValue(karpv1.CapacityTypeLabelKey, karpv1.CapacityTypeReserved)) - Expect(n.Labels).To(HaveKeyWithValue(v1.LabelCapacityReservationType, string(v1.CapacityReservationTypeDefault))) - Expect(n.Labels).To(HaveKeyWithValue(v1.LabelCapacityReservationID, xlargeCapacityReservationID)) - }) - // NOTE: We're not exercising capacity blocks because it isn't possible to provision them ad-hoc for the use in an - // integration test. - It("should schedule against a specific reservation type", func() { - selectors.Insert(v1.LabelCapacityReservationType) - pod := test.Pod(test.PodOptions{ - NodeRequirements: []corev1.NodeSelectorRequirement{ - { - Key: v1.LabelCapacityReservationType, - Operator: corev1.NodeSelectorOpIn, - Values: []string{string(v1.CapacityReservationTypeDefault)}, - }, - // NOTE: Continue to select the xlarge instance to ensure we can use the large instance for the fallback test. ODCR - // capacity eventual consistency is inconsistent between different services (e.g. DescribeCapacityReservations and - // RunInstances) so we've allocated enough to ensure that each test can make use of them without overlapping. - { - Key: corev1.LabelInstanceTypeStable, - Operator: corev1.NodeSelectorOpIn, - Values: []string{string(ec2types.InstanceTypeM5Xlarge)}, - }, - }, - }) - env.ExpectCreated(nodePool, nodeClass, pod) - - nc := env.EventuallyExpectLaunchedNodeClaimCount("==", 1)[0] - req, ok := lo.Find(nc.Spec.Requirements, func(req karpv1.NodeSelectorRequirementWithMinValues) bool { - return req.Key == v1.LabelCapacityReservationType - }) - Expect(ok).To(BeTrue()) - Expect(req.Values).To(ConsistOf(string(v1.CapacityReservationTypeDefault))) - - env.EventuallyExpectNodeClaimsReady(nc) - n := env.EventuallyExpectNodeCount("==", 1)[0] - Expect(n.Labels).To(HaveKeyWithValue(karpv1.CapacityTypeLabelKey, karpv1.CapacityTypeReserved)) - Expect(n.Labels).To(HaveKeyWithValue(v1.LabelCapacityReservationType, string(v1.CapacityReservationTypeDefault))) - Expect(n.Labels).To(HaveKeyWithValue(v1.LabelCapacityReservationID, xlargeCapacityReservationID)) - }) - It("should fall back when compatible capacity reservations are exhausted", func() { - // We create two pods with self anti-affinity and a node selector on a specific instance type. The anti-affinity term - // ensures that we must provision 2 nodes, and the node selector selects upon an instance type with a single reserved - // instance available. As such, we should create a reserved NodeClaim for one pod, and an on-demand NodeClaim for the - // other. - podLabels := map[string]string{"foo": "bar"} - pods := test.Pods(2, test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{ - Labels: podLabels, - }, - NodeRequirements: []corev1.NodeSelectorRequirement{{ - Key: corev1.LabelInstanceTypeStable, - Operator: corev1.NodeSelectorOpIn, - Values: []string{string(ec2types.InstanceTypeM5Large)}, - }}, - PodAntiRequirements: []corev1.PodAffinityTerm{{ - TopologyKey: corev1.LabelHostname, - LabelSelector: &metav1.LabelSelector{ - MatchLabels: podLabels, - }, - }}, - }) - env.ExpectCreated(nodePool, nodeClass, pods[0], pods[1]) - - reservedCount := 0 - for _, nc := range env.EventuallyExpectLaunchedNodeClaimCount("==", 2) { - req, ok := lo.Find(nc.Spec.Requirements, func(req karpv1.NodeSelectorRequirementWithMinValues) bool { - return req.Key == v1.LabelCapacityReservationID - }) - if ok { - reservedCount += 1 - Expect(req.Values).To(ConsistOf(largeCapacityReservationID)) - } - } - Expect(reservedCount).To(Equal(1)) - env.EventuallyExpectNodeCount("==", 2) - }) - }) - - Context("Interruptible Capacity Resverations", func() { - var sourceReservationID, interruptibleReservationID, xlargeReservationID string - BeforeAll(func() { - sourceReservationID, interruptibleReservationID = environmentaws.ExpectInterruptibleCapacityReservationCreated( - env.Context, - env.EC2API, - ec2types.InstanceTypeM5Large, - env.ZoneInfo[0].Zone, - 2, - 1, - nil, - ) - xlargeReservationID = environmentaws.ExpectCapacityReservationCreated( - env.Context, - env.EC2API, - ec2types.InstanceTypeM5Xlarge, - env.ZoneInfo[0].Zone, - 1, - nil, - nil, - ) - }) - AfterAll(func() { - environmentaws.ExpectInterruptibleAndSourceCapacityCanceled(env.Context, env.EC2API, sourceReservationID, interruptibleReservationID) - environmentaws.ExpectCapacityReservationsCanceled(env.Context, env.EC2API, xlargeReservationID) - }) - BeforeEach(func() { - nodeClass.Spec.CapacityReservationSelectorTerms = []v1.CapacityReservationSelectorTerm{ - {ID: sourceReservationID}, {ID: interruptibleReservationID}, - } - nodePool.Spec.Template.Spec.Requirements = []karpv1.NodeSelectorRequirementWithMinValues{ - { - Key: karpv1.CapacityTypeLabelKey, - Operator: corev1.NodeSelectorOpIn, - Values: []string{karpv1.CapacityTypeOnDemand, karpv1.CapacityTypeReserved}, - }, - { - Key: corev1.LabelOSStable, - Operator: corev1.NodeSelectorOpIn, - Values: []string{string(corev1.Linux)}, - }, - } - }) - - DescribeTable("should schedule against a specific reservation interruptibiltiy", func(interruptible bool) { - selectors.Insert(v1.LabelCapacityReservationInterruptible) - pod := test.Pod(test.PodOptions{ - NodeRequirements: []corev1.NodeSelectorRequirement{{ - Key: v1.LabelCapacityReservationInterruptible, - Operator: corev1.NodeSelectorOpIn, - Values: []string{strconv.FormatBool(interruptible)}, - }}, - }) - env.ExpectCreated(nodePool, nodeClass, pod) - - nc := env.EventuallyExpectLaunchedNodeClaimCount("==", 1)[0] - resReq, ok := lo.Find(nc.Spec.Requirements, func(req karpv1.NodeSelectorRequirementWithMinValues) bool { - return req.Key == v1.LabelCapacityReservationID - }) - Expect(ok).To(BeTrue()) - iReq, ok := lo.Find(nc.Spec.Requirements, func(req karpv1.NodeSelectorRequirementWithMinValues) bool { - return req.Key == v1.LabelCapacityReservationInterruptible - }) - Expect(ok).To(BeTrue()) - Expect(resReq.Values).To(ConsistOf(lo.Ternary(interruptible, interruptibleReservationID, sourceReservationID))) - Expect(iReq.Values).To(ConsistOf(strconv.FormatBool(interruptible))) - - env.EventuallyExpectNodeClaimsReady(nc) - n := env.EventuallyExpectNodeCount("==", 1)[0] - Expect(n.Labels).To(HaveKeyWithValue(v1.LabelCapacityReservationInterruptible, strconv.FormatBool(interruptible))) - }, - Entry("interruptible", true), - Entry("non-interruptible", false), - ) - It("should prioritize ODCR over IODCR if price is equal", func() { - pod := test.Pod(test.PodOptions{ - NodeRequirements: []corev1.NodeSelectorRequirement{{ - Key: corev1.LabelInstanceTypeStable, - Operator: corev1.NodeSelectorOpIn, - Values: []string{string(ec2types.InstanceTypeM5Large)}, - }}, - }) - env.ExpectCreated(nodePool, nodeClass, pod) - - nc := env.EventuallyExpectLaunchedNodeClaimCount("==", 1)[0] - req, ok := lo.Find(nc.Spec.Requirements, func(req karpv1.NodeSelectorRequirementWithMinValues) bool { - return req.Key == v1.LabelCapacityReservationID - }) - Expect(ok).To(BeTrue()) - // NodeClaim should have both reservations (but launch with ODCR) - Expect(req.Values).To(ConsistOf(sourceReservationID, interruptibleReservationID)) - - env.EventuallyExpectNodeClaimsReady(nc) - n := env.EventuallyExpectNodeCount("==", 1)[0] - Expect(n.Labels).To(HaveKeyWithValue(v1.LabelCapacityReservationInterruptible, "false")) - Expect(n.Labels).To(HaveKeyWithValue(v1.LabelCapacityReservationID, sourceReservationID)) - }) - It("should prioritize reservation with lower price", func() { - env.ExpectCreated(nodeClass) - nodeClass.Spec.CapacityReservationSelectorTerms = []v1.CapacityReservationSelectorTerm{ - {ID: xlargeReservationID}, {ID: interruptibleReservationID}, - } - env.ExpectUpdated(nodeClass) - - pod := test.Pod(test.PodOptions{ - NodeRequirements: []corev1.NodeSelectorRequirement{{ - Key: corev1.LabelInstanceTypeStable, - Operator: corev1.NodeSelectorOpIn, - Values: []string{string(ec2types.InstanceTypeM5Large), string(ec2types.InstanceTypeM5Xlarge)}, - }}, - }) - env.ExpectCreated(nodePool, pod) - - nc := env.EventuallyExpectLaunchedNodeClaimCount("==", 1)[0] - req, ok := lo.Find(nc.Spec.Requirements, func(req karpv1.NodeSelectorRequirementWithMinValues) bool { - return req.Key == v1.LabelCapacityReservationID - }) - Expect(ok).To(BeTrue()) - // NodeClaim should have both reservations (but launch in IODCR as its cheaper) - Expect(req.Values).To(ConsistOf(xlargeReservationID, interruptibleReservationID)) - - env.EventuallyExpectNodeClaimsReady(nc) - n := env.EventuallyExpectNodeCount("==", 1)[0] - Expect(n.Labels).To(HaveKeyWithValue(corev1.LabelInstanceTypeStable, string(ec2types.InstanceTypeM5Large))) - Expect(n.Labels).To(HaveKeyWithValue(v1.LabelCapacityReservationInterruptible, "true")) - }) - }) - It("should provision EFAs according to NodeClass configuration", func() { - pod := test.Pod(test.PodOptions{ - NodeRequirements: []corev1.NodeSelectorRequirement{ - {Key: v1.LabelInstanceCategory, Operator: corev1.NodeSelectorOpIn, Values: []string{"m"}}, - }, - }) - nodeClass.Spec.NetworkInterfaces = []*v1.NetworkInterface{ - {NetworkCardIndex: 0, DeviceIndex: 0, InterfaceType: v1.InterfaceTypeInterface}, - {NetworkCardIndex: 0, DeviceIndex: 1, InterfaceType: v1.InterfaceTypeEFAOnly}, - } - env.ExpectCreated(nodeClass, nodePool, pod) - env.EventuallyExpectHealthy(pod) - node := env.ExpectCreatedNodeCount("==", 1)[0] - - instance := env.GetInstance(node.Name) - networkInterfaces := instance.NetworkInterfaces - Expect(networkInterfaces).To(HaveLen(2)) - - for _, deviceIndex := range []int32{0, 1} { - networkInterface, found := lo.Find(networkInterfaces, func(i ec2types.InstanceNetworkInterface) bool { - Expect(i.Attachment).ToNot(BeNil()) - Expect(i.Attachment.DeviceIndex).ToNot(BeNil()) - return deviceIndex == lo.FromPtr(i.Attachment.DeviceIndex) - }) - Expect(found).To(Equal(true)) - - Expect(lo.FromPtr(networkInterface.InterfaceType)).To(Equal( - lo.Ternary(deviceIndex == 0, string(ec2types.NetworkInterfaceTypeInterface), string(ec2types.NetworkInterfaceTypeEfaOnly)), - )) - } - }) -}, - Entry("MinValuesPolicyBestEffort", options.MinValuesPolicyBestEffort), - Entry("MinValuesPolicyStrict", options.MinValuesPolicyStrict), -) - -var _ = Describe("Node Overlay", func() { - It("should provision the instance that is the cheepest based on a price adjustment node overlay applied", func() { - overlaiedInstanceType := "m7a.8xlarge" - pod := test.Pod() - nodeOverlay := test.NodeOverlay(v1alpha1.NodeOverlay{ - Spec: v1alpha1.NodeOverlaySpec{ - PriceAdjustment: lo.ToPtr("-99.99999999999%"), - Requirements: []v1alpha1.NodeSelectorRequirement{ - { - Key: corev1.LabelInstanceTypeStable, - Operator: corev1.NodeSelectorOpIn, - Values: []string{overlaiedInstanceType}, - }, - }, - }, - }) - env.ExpectCreated(nodePool, nodeClass, nodeOverlay, pod) - env.EventuallyExpectHealthy(pod) - node := env.EventuallyExpectInitializedNodeCount("==", 1) - - instanceType, foundInstanceType := node[0].Labels[corev1.LabelInstanceTypeStable] - Expect(foundInstanceType).To(BeTrue()) - Expect(instanceType).To(Equal(overlaiedInstanceType)) - }) - It("should provision the instance that is the cheepest based on a price override node overlay applied", func() { - overlaiedInstanceType := "c7a.8xlarge" - pod := test.Pod() - nodeOverlay := test.NodeOverlay(v1alpha1.NodeOverlay{ - Spec: v1alpha1.NodeOverlaySpec{ - Price: lo.ToPtr("0.0000000232"), - Requirements: []v1alpha1.NodeSelectorRequirement{ - { - Key: corev1.LabelInstanceTypeStable, - Operator: corev1.NodeSelectorOpIn, - Values: []string{overlaiedInstanceType}, - }, - }, - }, - }) - env.ExpectCreated(nodePool, nodeClass, nodeOverlay, pod) - env.EventuallyExpectHealthy(pod) - node := env.EventuallyExpectInitializedNodeCount("==", 1) - - instanceType, foundInstanceType := node[0].Labels[corev1.LabelInstanceTypeStable] - Expect(foundInstanceType).To(BeTrue()) - Expect(instanceType).To(Equal(overlaiedInstanceType)) - }) - It("should provision a node that matches hugepages resource requests", func() { - overlaiedInstanceType := "c7a.2xlarge" - pod := test.Pod(test.PodOptions{ - ResourceRequirements: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceCPU: test.RandomCPU(), - corev1.ResourceMemory: test.RandomMemory(), - corev1.ResourceName("hugepages-2Mi"): resource.MustParse("100Mi"), - }, - Limits: corev1.ResourceList{ - corev1.ResourceName("hugepages-2Mi"): resource.MustParse("100Mi"), - }, - }, - }) - nodeOverlay := test.NodeOverlay(v1alpha1.NodeOverlay{ - Spec: v1alpha1.NodeOverlaySpec{ - Requirements: []v1alpha1.NodeSelectorRequirement{ - { - Key: corev1.LabelInstanceTypeStable, - Operator: corev1.NodeSelectorOpIn, - Values: []string{overlaiedInstanceType}, - }, - }, - Capacity: corev1.ResourceList{ - corev1.ResourceName("hugepages-2Mi"): resource.MustParse("4Gi"), - }, - }, - }) - - content, err := os.ReadFile("testdata/hugepage_userdata_input.sh") - Expect(err).To(BeNil()) - nodeClass.Spec.UserData = lo.ToPtr(string(content)) - - env.ExpectCreated(nodePool, nodeClass, nodeOverlay, pod) - env.EventuallyExpectHealthy(pod) - node := env.EventuallyExpectInitializedNodeCount("==", 1) - - instanceType, foundInstanceType := node[0].Labels[corev1.LabelInstanceTypeStable] - Expect(foundInstanceType).To(BeTrue()) - Expect(instanceType).To(Equal(overlaiedInstanceType)) - }) -}) - -func ephemeralInitContainer(requirements corev1.ResourceRequirements) corev1.Container { - return corev1.Container{ - Image: environmentaws.EphemeralInitContainerImage, - Command: []string{"/bin/sh"}, - Args: []string{"-c", "sleep 5"}, - Resources: requirements, - } -}