diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d38cb66e5c..a6b03c8b762 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -86,6 +86,7 @@ To learn more about active deprecations, we recommend checking [GitHub Discussio - **General**: Check updated status for Fallback condition instead of ScaledObject ([#7488](https://github.com/kedacore/keda/issues/7488)) - **General**: Fix int64 overflow in milli-quantity conversion for very large metric values ([#7441](https://github.com/kedacore/keda/issues/7441)) - **General**: Fix ScaledObject admission webhook to return validation error from `verifyReplicaCount`, preventing invalid ScaledObjects from being created ([#5954](https://github.com/kedacore/keda/issues/5954)) +- **General**: Fix ScaledObject created with paused annotation not properly transitioning to active when annotation is removed ([#6421](https://github.com/kedacore/keda/issues/6421)) - **Cron Scaler**: Fix metric name generation so cron expressions with comma-separated values no longer produce invalid metric names ([#7448](https://github.com/kedacore/keda/issues/7448)) - **External Scaler**: Fix context cancellation handling in `waitForState` of external scaler ([#7542](https://github.com/kedacore/keda/issues/7542)) - **Forgejo Scaler**: Limit HTTP error response logging ([#7469](https://github.com/kedacore/keda/pull/7469)) diff --git a/controllers/keda/scaledobject_controller.go b/controllers/keda/scaledobject_controller.go index 76b9add1a45..7356b5bd5e7 100755 --- a/controllers/keda/scaledobject_controller.go +++ b/controllers/keda/scaledobject_controller.go @@ -288,7 +288,18 @@ func (r *ScaledObjectReconciler) reconcileScaledObject(ctx context.Context, logg } case isPausedInStatus: unpausedMessage := "pause annotation removed for ScaledObject" + // Write unpaused status FIRST before any operations that might trigger new + // reconciles or start the scale loop. This mirrors the pause path which writes + // Paused=True before stopping the scale loop. Without this, a race condition + // exists where the scale loop (started later in this reconcile via + // requestScaleLoop) reads the stale Paused=True condition and overwrites it + // back via a merge patch that replaces the entire conditions array. + // See: https://github.com/kedacore/keda/issues/6421 + logger.Info("Setting Unpaused condition before starting scale loop") conditions.SetPausedCondition(metav1.ConditionFalse, "ScaledObjectUnpaused", unpausedMessage) + if err := kedastatus.SetStatusConditions(ctx, r.Client, logger, scaledObject, conditions); err != nil { + return "failed to update unpaused status", err + } r.EventEmitter.Emit(scaledObject, scaledObject.Namespace, corev1.EventTypeNormal, eventingv1alpha1.ScaledObjectUnpausedType, eventreason.ScaledObjectUnpaused, unpausedMessage) } diff --git a/controllers/keda/scaledobject_controller_test.go b/controllers/keda/scaledobject_controller_test.go index e77a49a4759..255ce9a0efc 100644 --- a/controllers/keda/scaledobject_controller_test.go +++ b/controllers/keda/scaledobject_controller_test.go @@ -1804,6 +1804,100 @@ var _ = Describe("ScaledObjectController", func() { }).Should(HaveOccurred()) }) + // Fix issue 6421 + It("scaledobject created with paused annotation becomes active when annotation is removed", func() { + // Create the scaling target. + deploymentName := "initially-paused-then-unpaused" + soName := "so-" + deploymentName + err := k8sClient.Create(context.Background(), generateDeployment(deploymentName)) + Expect(err).ToNot(HaveOccurred()) + + // Create the ScaledObject with the paused annotation already set. + so := &kedav1alpha1.ScaledObject{ + ObjectMeta: metav1.ObjectMeta{ + Name: soName, + Namespace: "default", + Annotations: map[string]string{ + kedav1alpha1.PausedAnnotation: "true", + }, + }, + Spec: kedav1alpha1.ScaledObjectSpec{ + ScaleTargetRef: &kedav1alpha1.ScaleTarget{ + Name: deploymentName, + }, + Advanced: &kedav1alpha1.AdvancedConfig{ + HorizontalPodAutoscalerConfig: &kedav1alpha1.HorizontalPodAutoscalerConfig{}, + }, + Triggers: []kedav1alpha1.ScaleTriggers{ + { + Type: "cron", + Metadata: map[string]string{ + "timezone": "UTC", + "start": "0 * * * *", + "end": "1 * * * *", + "desiredReplicas": "1", + }, + }, + }, + }, + } + pollingInterval := int32(5) + so.Spec.PollingInterval = &pollingInterval + err = k8sClient.Create(context.Background(), so) + Expect(err).ToNot(HaveOccurred()) + + // Wait for Paused condition to become True. + Eventually(func() metav1.ConditionStatus { + err := k8sClient.Get(context.Background(), types.NamespacedName{Name: soName, Namespace: "default"}, so) + if err != nil { + return metav1.ConditionFalse + } + return so.Status.Conditions.GetPausedCondition().Status + }).WithTimeout(1 * time.Minute).WithPolling(2 * time.Second).Should(Equal(metav1.ConditionTrue)) + + // HPA should NOT exist while paused. + hpa := &autoscalingv2.HorizontalPodAutoscaler{} + Consistently(func() bool { + err := k8sClient.Get(context.Background(), types.NamespacedName{Name: fmt.Sprintf("keda-hpa-%s", soName), Namespace: "default"}, hpa) + return errors.IsNotFound(err) + }, 5*time.Second, 1*time.Second).Should(BeTrue()) + + // Remove the paused annotation to unpause. + Eventually(func() error { + err = k8sClient.Get(context.Background(), types.NamespacedName{Name: soName, Namespace: "default"}, so) + Expect(err).ToNot(HaveOccurred()) + annotations := so.GetAnnotations() + delete(annotations, kedav1alpha1.PausedAnnotation) + so.SetAnnotations(annotations) + return k8sClient.Update(context.Background(), so) + }).WithTimeout(1 * time.Minute).WithPolling(5 * time.Second).ShouldNot(HaveOccurred()) + testLogger.Info("paused annotation removed") + + // Paused condition should become False. + Eventually(func() metav1.ConditionStatus { + err := k8sClient.Get(context.Background(), types.NamespacedName{Name: soName, Namespace: "default"}, so) + if err != nil { + return metav1.ConditionTrue + } + return so.Status.Conditions.GetPausedCondition().Status + }).WithTimeout(1 * time.Minute).WithPolling(2 * time.Second).Should(Equal(metav1.ConditionFalse)) + + // HPA should be created after unpausing. + Eventually(func() error { + return k8sClient.Get(context.Background(), types.NamespacedName{Name: fmt.Sprintf("keda-hpa-%s", soName), Namespace: "default"}, hpa) + }).WithTimeout(1 * time.Minute).WithPolling(2 * time.Second).ShouldNot(HaveOccurred()) + + // Crucially: Paused condition must STAY False and not flip back to True. + // This is the core assertion for issue #6421. + Consistently(func() metav1.ConditionStatus { + err := k8sClient.Get(context.Background(), types.NamespacedName{Name: soName, Namespace: "default"}, so) + if err != nil { + return metav1.ConditionTrue + } + return so.Status.Conditions.GetPausedCondition().Status + }, 15*time.Second, 2*time.Second).Should(Equal(metav1.ConditionFalse)) + }) + // Fix issue 5281 It("reconciles scaledobject when hpa spec is changed", func() { var (