diff --git a/pkg/frontend/admin_openshiftcluster_vmresize_pre_validation.go b/pkg/frontend/admin_openshiftcluster_vmresize_pre_validation.go index 56e1c5b043f..422fb387cc3 100644 --- a/pkg/frontend/admin_openshiftcluster_vmresize_pre_validation.go +++ b/pkg/frontend/admin_openshiftcluster_vmresize_pre_validation.go @@ -16,6 +16,8 @@ import ( "github.com/go-chi/chi/v5" "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" + configv1 "github.com/openshift/api/config/v1" operatorv1 "github.com/openshift/api/operator/v1" @@ -104,10 +106,23 @@ func (f *frontend) _getPreResizeControlPlaneVMsValidation( } } + if err := k.CheckAPIServerReadyz(ctx); err != nil { + return nil, api.NewCloudError( + http.StatusInternalServerError, + api.CloudErrorCodeInternalServerError, "kube-apiserver", + fmt.Sprintf("API server is reporting a non-ready status: %v", err)) + } + var wg sync.WaitGroup wg.Go(func() { collect(f.validateVMSKU(ctx, doc, subscriptionDoc, desiredVMSize, log)) }) - wg.Go(func() { collect(validateAPIServerHealth(ctx, k)) }) + wg.Go(func() { + if err := validateAPIServerHealth(ctx, k); err != nil { + collect(err) + return + } + collect(validateAPIServerPods(ctx, k)) + }) wg.Go(func() { collect(validateEtcdHealth(ctx, k)) }) wg.Go(func() { collect(validateClusterSP(ctx, k)) }) @@ -124,7 +139,7 @@ func (f *frontend) _getPreResizeControlPlaneVMsValidation( } } - return json.Marshal("All pre-flight checks passed") + return json.Marshal(map[string]string{"status": "passed"}) } // defaultValidateResizeQuota creates an FP-authorized compute usage client and @@ -218,8 +233,9 @@ func quotaCheckDisabled(_ context.Context, _ env.Interface, _ *api.SubscriptionD return nil } -// validateAPIServerHealth verifies that the kube-apiserver ClusterOperator is -// healthy (Available=True, Progressing=False, Degraded=False). +// validateAPIServerHealth verifies that the kube-apiserver ClusterOperator is healthy +// (Available=True, Progressing=False, Degraded=False). +// Note: API server reachability is checked earlier via CheckAPIServerReadyz func validateAPIServerHealth(ctx context.Context, k adminactions.KubeActions) error { rawCO, err := k.KubeGet(ctx, "ClusterOperator.config.openshift.io", "", "kube-apiserver") if err != nil { @@ -241,13 +257,83 @@ func validateAPIServerHealth(ctx context.Context, k adminactions.KubeActions) er return api.NewCloudError( http.StatusConflict, api.CloudErrorCodeRequestNotAllowed, "kube-apiserver", - fmt.Sprintf("kube-apiserver is not healthy: %s. Resize is not safe while the API server is degraded.", + fmt.Sprintf("kube-apiserver is not healthy: %s. Resize is not safe while the API server is unhealthy.", clusteroperators.OperatorStatusText(&co))) } return nil } +func validateAPIServerPods(ctx context.Context, k adminactions.KubeActions) error { + const ( + kubeAPIServerNamespace = "openshift-kube-apiserver" + kubeAPIServerAppLabel = "openshift-kube-apiserver" + ) + + rawPods, err := k.KubeList(ctx, "Pod", kubeAPIServerNamespace) + if err != nil { + return api.NewCloudError( + http.StatusInternalServerError, + api.CloudErrorCodeInternalServerError, "kube-apiserver-pods", + fmt.Sprintf("Failed to list pods in %s namespace: %v", kubeAPIServerNamespace, err)) + } + + var podList corev1.PodList + if err := json.Unmarshal(rawPods, &podList); err != nil { + return api.NewCloudError( + http.StatusInternalServerError, + api.CloudErrorCodeInternalServerError, "kube-apiserver-pods", + fmt.Sprintf("Failed to parse pod list: %v", err)) + } + + var apiServerPodCount int + var unhealthyPods []string + for _, pod := range podList.Items { + if pod.Labels["app"] != kubeAPIServerAppLabel { + continue + } + + apiServerPodCount++ + + if healthy, reason := isPodHealthy(&pod); !healthy { + unhealthyPods = append(unhealthyPods, fmt.Sprintf("%s (%s)", pod.Name, reason)) + } + } + + if apiServerPodCount != api.ControlPlaneNodeCount { + return api.NewCloudError( + http.StatusConflict, + api.CloudErrorCodeRequestNotAllowed, "kube-apiserver-pods", + fmt.Sprintf("Expected %d kube-apiserver pods, found %d. Resize is not safe without full API server redundancy.", + api.ControlPlaneNodeCount, apiServerPodCount)) + } + + if len(unhealthyPods) > 0 { + return api.NewCloudError( + http.StatusConflict, + api.CloudErrorCodeRequestNotAllowed, "kube-apiserver-pods", + fmt.Sprintf("Unhealthy kube-apiserver pods: %v. Resize is not safe without full API server redundancy.", + unhealthyPods)) + } + + return nil +} + +func isPodHealthy(pod *corev1.Pod) (healthy bool, reason string) { + if pod.Status.Phase != corev1.PodRunning { + return false, fmt.Sprintf("phase: %s", pod.Status.Phase) + } + for _, cond := range pod.Status.Conditions { + if cond.Type == corev1.PodReady { + if cond.Status != corev1.ConditionTrue { + return false, "not ready" + } + return true, "" + } + } + return false, "Ready condition not found" +} + // validateEtcdHealth verifies that the etcd ClusterOperator is healthy. // Resizing takes a master offline, so all etcd members must be healthy. func validateEtcdHealth(ctx context.Context, k adminactions.KubeActions) error { diff --git a/pkg/frontend/admin_openshiftcluster_vmresize_pre_validation_test.go b/pkg/frontend/admin_openshiftcluster_vmresize_pre_validation_test.go index f18175ceba7..a46cb4a4d29 100644 --- a/pkg/frontend/admin_openshiftcluster_vmresize_pre_validation_test.go +++ b/pkg/frontend/admin_openshiftcluster_vmresize_pre_validation_test.go @@ -14,6 +14,7 @@ import ( "github.com/sirupsen/logrus" "go.uber.org/mock/gomock" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v7" @@ -77,11 +78,52 @@ func healthyEtcdJSON() []byte { }) } +func fakeAPIServerPodListJSON(pods []corev1.Pod) []byte { + podList := corev1.PodList{Items: pods} + b, _ := json.Marshal(podList) + return b +} + +func healthyAPIServerPod(name, nodeName string) corev1.Pod { + return corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "openshift-kube-apiserver", + Labels: map[string]string{"app": "openshift-kube-apiserver"}, + }, + Spec: corev1.PodSpec{ + NodeName: nodeName, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + Conditions: []corev1.PodCondition{ + {Type: corev1.PodReady, Status: corev1.ConditionTrue}, + }, + }, + } +} + +func healthyAPIServerPodsJSON() []byte { + return fakeAPIServerPodListJSON([]corev1.Pod{ + healthyAPIServerPod("kube-apiserver-master-0", "master-0"), + healthyAPIServerPod("kube-apiserver-master-1", "master-1"), + healthyAPIServerPod("kube-apiserver-master-2", "master-2"), + }) +} + func allKubeChecksHealthyMock(k *mock_adminactions.MockKubeActions) { + k.EXPECT(). + CheckAPIServerReadyz(gomock.Any()). + Return(nil). + AnyTimes() k.EXPECT(). KubeGet(gomock.Any(), "ClusterOperator.config.openshift.io", "", "kube-apiserver"). Return(healthyKubeAPIServerJSON(), nil). AnyTimes() + k.EXPECT(). + KubeList(gomock.Any(), "Pod", "openshift-kube-apiserver"). + Return(healthyAPIServerPodsJSON(), nil). + AnyTimes() k.EXPECT(). KubeGet(gomock.Any(), "ClusterOperator.config.openshift.io", "", "etcd"). Return(healthyEtcdJSON(), nil). @@ -160,7 +202,7 @@ func TestPreResizeControlPlaneVMsValidation(t *testing.T) { }, kubeMocks: allKubeChecksHealthyMock, wantStatusCode: http.StatusOK, - wantResponse: []byte(`"All pre-flight checks passed"` + "\n"), + wantResponse: []byte(`{"status":"passed"}` + "\n"), }, { name: "missing vmSize parameter", @@ -374,6 +416,45 @@ func TestPreResizeControlPlaneVMsValidation(t *testing.T) { wantStatusCode: http.StatusBadRequest, wantError: `400: InvalidParameter: : Pre-flight validation failed. Details: InvalidParameter: vmSize: The selected SKU 'Standard_D8s_v3' is restricted in region 'eastus' for selected subscription`, }, + { + name: "API server unreachable", + resourceID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + vmSize: "Standard_D8s_v3", + fixture: func(f *testdatabase.Fixture) { + f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(testdatabase.GetResourcePath(mockSubID, "resourceName")), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + Location: "eastus", + Properties: api.OpenShiftClusterProperties{ + MasterProfile: api.MasterProfile{ + VMSize: api.VMSizeStandardD8sV3, + }, + ClusterProfile: api.ClusterProfile{ + ResourceGroupID: fmt.Sprintf("/subscriptions/%s/resourceGroups/test-cluster", mockSubID), + }, + }, + }, + }) + f.AddSubscriptionDocuments(&api.SubscriptionDocument{ + ID: mockSubID, + Subscription: &api.Subscription{ + State: api.SubscriptionStateRegistered, + Properties: &api.SubscriptionProperties{ + TenantID: mockTenantID, + }, + }, + }) + }, + mocks: func(tt *test, a *mock_adminactions.MockAzureActions) {}, + kubeMocks: func(k *mock_adminactions.MockKubeActions) { + k.EXPECT(). + CheckAPIServerReadyz(gomock.Any()). + Return(fmt.Errorf("connection refused")) + }, + wantStatusCode: http.StatusInternalServerError, + wantError: `500: InternalServerError: kube-apiserver: API server is reporting a non-ready status: connection refused`, + }, } { t.Run(tt.name, func(t *testing.T) { ti := newTestInfra(t).WithSubscriptions().WithOpenShiftClusters() @@ -754,7 +835,7 @@ func TestValidateAPIServerHealth(t *testing.T) { {Type: configv1.OperatorDegraded, Status: configv1.ConditionTrue}, }), nil) }, - wantErr: "409: RequestNotAllowed: kube-apiserver: kube-apiserver is not healthy: kube-apiserver Available=True, Progressing=False. Resize is not safe while the API server is degraded.", + wantErr: "409: RequestNotAllowed: kube-apiserver: kube-apiserver is not healthy: kube-apiserver Available=True, Progressing=False, Degraded=True. Resize is not safe while the API server is unhealthy.", }, { name: "kube-apiserver unavailable", @@ -767,7 +848,7 @@ func TestValidateAPIServerHealth(t *testing.T) { {Type: configv1.OperatorDegraded, Status: configv1.ConditionFalse}, }), nil) }, - wantErr: "409: RequestNotAllowed: kube-apiserver: kube-apiserver is not healthy: kube-apiserver Available=False, Progressing=True. Resize is not safe while the API server is degraded.", + wantErr: "409: RequestNotAllowed: kube-apiserver: kube-apiserver is not healthy: kube-apiserver Available=False, Progressing=True, Degraded=False. Resize is not safe while the API server is unhealthy.", }, { name: "KubeGet returns error", @@ -828,7 +909,7 @@ func TestValidateEtcdHealth(t *testing.T) { {Type: configv1.OperatorDegraded, Status: configv1.ConditionTrue}, }), nil) }, - wantErr: "409: RequestNotAllowed: etcd: etcd is not healthy: etcd Available=True, Progressing=False. Resize is not safe while etcd quorum is at risk.", + wantErr: "409: RequestNotAllowed: etcd: etcd is not healthy: etcd Available=True, Progressing=False, Degraded=True. Resize is not safe while etcd quorum is at risk.", }, { name: "etcd unavailable", @@ -841,7 +922,7 @@ func TestValidateEtcdHealth(t *testing.T) { {Type: configv1.OperatorDegraded, Status: configv1.ConditionFalse}, }), nil) }, - wantErr: "409: RequestNotAllowed: etcd: etcd is not healthy: etcd Available=False, Progressing=True. Resize is not safe while etcd quorum is at risk.", + wantErr: "409: RequestNotAllowed: etcd: etcd is not healthy: etcd Available=False, Progressing=True, Degraded=False. Resize is not safe while etcd quorum is at risk.", }, { name: "KubeGet returns error", @@ -874,3 +955,184 @@ func TestValidateEtcdHealth(t *testing.T) { }) } } + +func TestValidateAPIServerPods(t *testing.T) { + ctx := context.Background() + + for _, tt := range []struct { + name string + mocks func(*mock_adminactions.MockKubeActions) + wantErr string + }{ + { + name: "all pods healthy", + mocks: func(k *mock_adminactions.MockKubeActions) { + k.EXPECT(). + KubeList(gomock.Any(), "Pod", "openshift-kube-apiserver"). + Return(healthyAPIServerPodsJSON(), nil) + }, + }, + { + name: "KubeList returns error", + mocks: func(k *mock_adminactions.MockKubeActions) { + k.EXPECT(). + KubeList(gomock.Any(), "Pod", "openshift-kube-apiserver"). + Return(nil, fmt.Errorf("connection refused")) + }, + wantErr: "500: InternalServerError: kube-apiserver-pods: Failed to list pods in openshift-kube-apiserver namespace: connection refused", + }, + { + name: "KubeList returns invalid JSON", + mocks: func(k *mock_adminactions.MockKubeActions) { + k.EXPECT(). + KubeList(gomock.Any(), "Pod", "openshift-kube-apiserver"). + Return([]byte(`{invalid`), nil) + }, + wantErr: "500: InternalServerError: kube-apiserver-pods: Failed to parse pod list: invalid character 'i' looking for beginning of object key string", + }, + { + name: "only 2 apiserver pods", + mocks: func(k *mock_adminactions.MockKubeActions) { + k.EXPECT(). + KubeList(gomock.Any(), "Pod", "openshift-kube-apiserver"). + Return(fakeAPIServerPodListJSON([]corev1.Pod{ + healthyAPIServerPod("kube-apiserver-master-0", "master-0"), + healthyAPIServerPod("kube-apiserver-master-1", "master-1"), + }), nil) + }, + wantErr: "409: RequestNotAllowed: kube-apiserver-pods: Expected 3 kube-apiserver pods, found 2. Resize is not safe without full API server redundancy.", + }, + { + name: "4 apiserver pods", + mocks: func(k *mock_adminactions.MockKubeActions) { + k.EXPECT(). + KubeList(gomock.Any(), "Pod", "openshift-kube-apiserver"). + Return(fakeAPIServerPodListJSON([]corev1.Pod{ + healthyAPIServerPod("kube-apiserver-master-0", "master-0"), + healthyAPIServerPod("kube-apiserver-master-1", "master-1"), + healthyAPIServerPod("kube-apiserver-master-2", "master-2"), + healthyAPIServerPod("kube-apiserver-master-3", "master-3"), + }), nil) + }, + wantErr: "409: RequestNotAllowed: kube-apiserver-pods: Expected 3 kube-apiserver pods, found 4. Resize is not safe without full API server redundancy.", + }, + { + name: "one pod not running", + mocks: func(k *mock_adminactions.MockKubeActions) { + pods := []corev1.Pod{ + healthyAPIServerPod("kube-apiserver-master-0", "master-0"), + healthyAPIServerPod("kube-apiserver-master-1", "master-1"), + { + ObjectMeta: metav1.ObjectMeta{ + Name: "kube-apiserver-master-2", + Namespace: "openshift-kube-apiserver", + Labels: map[string]string{"app": "openshift-kube-apiserver"}, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodPending, + }, + }, + } + k.EXPECT(). + KubeList(gomock.Any(), "Pod", "openshift-kube-apiserver"). + Return(fakeAPIServerPodListJSON(pods), nil) + }, + wantErr: "409: RequestNotAllowed: kube-apiserver-pods: Unhealthy kube-apiserver pods: [kube-apiserver-master-2 (phase: Pending)]. Resize is not safe without full API server redundancy.", + }, + { + name: "one pod not ready", + mocks: func(k *mock_adminactions.MockKubeActions) { + pods := []corev1.Pod{ + healthyAPIServerPod("kube-apiserver-master-0", "master-0"), + healthyAPIServerPod("kube-apiserver-master-1", "master-1"), + { + ObjectMeta: metav1.ObjectMeta{ + Name: "kube-apiserver-master-2", + Namespace: "openshift-kube-apiserver", + Labels: map[string]string{"app": "openshift-kube-apiserver"}, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + Conditions: []corev1.PodCondition{ + {Type: corev1.PodReady, Status: corev1.ConditionFalse}, + }, + }, + }, + } + k.EXPECT(). + KubeList(gomock.Any(), "Pod", "openshift-kube-apiserver"). + Return(fakeAPIServerPodListJSON(pods), nil) + }, + wantErr: "409: RequestNotAllowed: kube-apiserver-pods: Unhealthy kube-apiserver pods: [kube-apiserver-master-2 (not ready)]. Resize is not safe without full API server redundancy.", + }, + { + name: "multiple unhealthy pods", + mocks: func(k *mock_adminactions.MockKubeActions) { + pods := []corev1.Pod{ + healthyAPIServerPod("kube-apiserver-master-0", "master-0"), + { + ObjectMeta: metav1.ObjectMeta{ + Name: "kube-apiserver-master-1", + Namespace: "openshift-kube-apiserver", + Labels: map[string]string{"app": "openshift-kube-apiserver"}, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodFailed, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "kube-apiserver-master-2", + Namespace: "openshift-kube-apiserver", + Labels: map[string]string{"app": "openshift-kube-apiserver"}, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + Conditions: []corev1.PodCondition{ + {Type: corev1.PodReady, Status: corev1.ConditionFalse}, + }, + }, + }, + } + k.EXPECT(). + KubeList(gomock.Any(), "Pod", "openshift-kube-apiserver"). + Return(fakeAPIServerPodListJSON(pods), nil) + }, + wantErr: "409: RequestNotAllowed: kube-apiserver-pods: Unhealthy kube-apiserver pods: [kube-apiserver-master-1 (phase: Failed) kube-apiserver-master-2 (not ready)]. Resize is not safe without full API server redundancy.", + }, + { + name: "filters non-apiserver pods", + mocks: func(k *mock_adminactions.MockKubeActions) { + pods := []corev1.Pod{ + healthyAPIServerPod("kube-apiserver-master-0", "master-0"), + healthyAPIServerPod("kube-apiserver-master-1", "master-1"), + healthyAPIServerPod("kube-apiserver-master-2", "master-2"), + { + ObjectMeta: metav1.ObjectMeta{ + Name: "some-other-pod", + Namespace: "openshift-kube-apiserver", + Labels: map[string]string{"app": "other-app"}, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + }, + }, + } + k.EXPECT(). + KubeList(gomock.Any(), "Pod", "openshift-kube-apiserver"). + Return(fakeAPIServerPodListJSON(pods), nil) + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + k := mock_adminactions.NewMockKubeActions(controller) + tt.mocks(k) + + err := validateAPIServerPods(ctx, k) + utilerror.AssertErrorMessage(t, err, tt.wantErr) + }) + } +} diff --git a/pkg/frontend/adminactions/kubeactions.go b/pkg/frontend/adminactions/kubeactions.go index 54271b99cc4..ed0b101a985 100644 --- a/pkg/frontend/adminactions/kubeactions.go +++ b/pkg/frontend/adminactions/kubeactions.go @@ -46,6 +46,7 @@ type KubeActions interface { // Fetch top pods and nodes metrics TopPods(ctx context.Context, restConfig *restclient.Config, allNamespaces bool) ([]PodMetrics, error) TopNodes(ctx context.Context, restConfig *restclient.Config) ([]NodeMetrics, error) + CheckAPIServerReadyz(ctx context.Context) error } type kubeActions struct { @@ -187,3 +188,11 @@ func (k *kubeActions) KubeDelete(ctx context.Context, groupKind, namespace, name return k.dyn.Resource(gvr).Namespace(namespace).Delete(ctx, name, resourceDeleteOptions) } + +func (k *kubeActions) CheckAPIServerReadyz(ctx context.Context) error { + _, err := k.kubecli.Discovery().RESTClient().Get().AbsPath("/readyz").Do(ctx).Raw() + if err != nil { + return fmt.Errorf("API server readyz check failed: %w", err) + } + return nil +} diff --git a/pkg/mimo/steps/cluster/apiserver_is_up_test.go b/pkg/mimo/steps/cluster/apiserver_is_up_test.go index e6ede0e0475..b1eae3b5be3 100644 --- a/pkg/mimo/steps/cluster/apiserver_is_up_test.go +++ b/pkg/mimo/steps/cluster/apiserver_is_up_test.go @@ -56,11 +56,15 @@ func TestAPIServerIsUp(t *testing.T) { Type: configv1.OperatorProgressing, Status: configv1.ConditionTrue, }, + { + Type: configv1.OperatorDegraded, + Status: configv1.ConditionFalse, + }, }, }, }, }, - wantErr: `TransientError: kube-apiserver Available=False, Progressing=True`, + wantErr: `TransientError: kube-apiserver Available=False, Progressing=True, Degraded=False`, }, { name: "ready", diff --git a/pkg/util/clusteroperators/isavailable.go b/pkg/util/clusteroperators/isavailable.go index 6c8d96beeaa..6ebd417012d 100644 --- a/pkg/util/clusteroperators/isavailable.go +++ b/pkg/util/clusteroperators/isavailable.go @@ -29,7 +29,10 @@ func OperatorStatusText(operator *configv1.ClusterOperator) string { for _, cond := range operator.Status.Conditions { m[cond.Type] = cond.Status } - return fmt.Sprintf("%s %s=%s, %s=%s", operator.Name, - configv1.OperatorAvailable, m[configv1.OperatorAvailable], configv1.OperatorProgressing, m[configv1.OperatorProgressing], + + return fmt.Sprintf("%s %s=%s, %s=%s, %s=%s", operator.Name, + configv1.OperatorAvailable, m[configv1.OperatorAvailable], + configv1.OperatorProgressing, m[configv1.OperatorProgressing], + configv1.OperatorDegraded, m[configv1.OperatorDegraded], ) } diff --git a/pkg/util/clusteroperators/isavailable_test.go b/pkg/util/clusteroperators/isavailable_test.go index a1e9f631f16..d6e1d31f873 100644 --- a/pkg/util/clusteroperators/isavailable_test.go +++ b/pkg/util/clusteroperators/isavailable_test.go @@ -69,31 +69,43 @@ func TestOperatorStatusText(t *testing.T) { name string availableCondition configv1.ConditionStatus progressingCondition configv1.ConditionStatus + degradedCondition configv1.ConditionStatus want string }{ { - name: "Available && Progressing; not available", + name: "Available && Progressing && !Degraded", availableCondition: configv1.ConditionTrue, progressingCondition: configv1.ConditionTrue, - want: "server Available=True, Progressing=True", + degradedCondition: configv1.ConditionFalse, + want: "server Available=True, Progressing=True, Degraded=False", }, { - name: "Available && !Progressing; available", + name: "Available && !Progressing && !Degraded; healthy", availableCondition: configv1.ConditionTrue, progressingCondition: configv1.ConditionFalse, - want: "server Available=True, Progressing=False", + degradedCondition: configv1.ConditionFalse, + want: "server Available=True, Progressing=False, Degraded=False", }, { - name: "!Available && Progressing; not available", + name: "!Available && Progressing && !Degraded", availableCondition: configv1.ConditionFalse, progressingCondition: configv1.ConditionTrue, - want: "server Available=False, Progressing=True", + degradedCondition: configv1.ConditionFalse, + want: "server Available=False, Progressing=True, Degraded=False", }, { - name: "!Available && !Progressing; not available", + name: "!Available && !Progressing && !Degraded", availableCondition: configv1.ConditionFalse, progressingCondition: configv1.ConditionFalse, - want: "server Available=False, Progressing=False", + degradedCondition: configv1.ConditionFalse, + want: "server Available=False, Progressing=False, Degraded=False", + }, + { + name: "Available && !Progressing && Degraded", + availableCondition: configv1.ConditionTrue, + progressingCondition: configv1.ConditionFalse, + degradedCondition: configv1.ConditionTrue, + want: "server Available=True, Progressing=False, Degraded=True", }, } { operator := &configv1.ClusterOperator{ @@ -110,12 +122,16 @@ func TestOperatorStatusText(t *testing.T) { Type: configv1.OperatorProgressing, Status: tt.progressingCondition, }, + { + Type: configv1.OperatorDegraded, + Status: tt.degradedCondition, + }, }, }, } - available := OperatorStatusText(operator) - if available != tt.want { - t.Error(available) + got := OperatorStatusText(operator) + if got != tt.want { + t.Errorf("%s: OperatorStatusText() = %q, want %q", tt.name, got, tt.want) } } } diff --git a/pkg/util/mocks/adminactions/kubeactions.go b/pkg/util/mocks/adminactions/kubeactions.go index 06cca7b1e17..a4df7db16f6 100644 --- a/pkg/util/mocks/adminactions/kubeactions.go +++ b/pkg/util/mocks/adminactions/kubeactions.go @@ -76,6 +76,20 @@ func (mr *MockKubeActionsMockRecorder) ApproveCsr(ctx, csrName any) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ApproveCsr", reflect.TypeOf((*MockKubeActions)(nil).ApproveCsr), ctx, csrName) } +// CheckAPIServerReadyz mocks base method. +func (m *MockKubeActions) CheckAPIServerReadyz(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CheckAPIServerReadyz", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// CheckAPIServerReadyz indicates an expected call of CheckAPIServerReadyz. +func (mr *MockKubeActionsMockRecorder) CheckAPIServerReadyz(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckAPIServerReadyz", reflect.TypeOf((*MockKubeActions)(nil).CheckAPIServerReadyz), ctx) +} + // CordonNode mocks base method. func (m *MockKubeActions) CordonNode(ctx context.Context, nodeName string, unschedulable bool) error { m.ctrl.T.Helper()