diff --git a/fulfillment-service/internal/controllers/tenant/tenant_reconciler_function.go b/fulfillment-service/internal/controllers/tenant/tenant_reconciler_function.go index d2e8ea6d06..4829e554be 100644 --- a/fulfillment-service/internal/controllers/tenant/tenant_reconciler_function.go +++ b/fulfillment-service/internal/controllers/tenant/tenant_reconciler_function.go @@ -211,6 +211,9 @@ func (t *task) update(ctx context.Context) error { if err := t.ensureVaultNamespace(ctx); err != nil { return err } + if t.tenant.GetStatus().GetState() == privatev1.TenantState_TENANT_STATE_FAILED { + return nil + } if err := t.ensureDefaultNetworking(ctx); err != nil { return err } @@ -257,6 +260,9 @@ func (t *task) syncToIDP(ctx context.Context) error { if err := t.ensureVaultNamespace(ctx); err != nil { return err } + if t.tenant.GetStatus().GetState() == privatev1.TenantState_TENANT_STATE_FAILED { + return nil + } if err := t.persistBreakGlassSecret(ctx); err != nil { t.r.logger.ErrorContext(ctx, "Failed to persist break-glass credentials secret", @@ -709,7 +715,11 @@ func (t *task) ensureVaultNamespace(ctx context.Context) error { slog.String("tenant_name", tenantName), slog.Any("error", err), ) - return fmt.Errorf("failed to provision vault namespace: %w", err) + t.updateCondition(condType, privatev1.ConditionStatus_CONDITION_STATUS_FALSE, + "ProvisionFailed", fmt.Sprintf("Failed to provision vault namespace: %v", err)) + t.tenant.GetStatus().SetState(privatev1.TenantState_TENANT_STATE_FAILED) + t.tenant.GetStatus().SetMessage(fmt.Sprintf("Vault namespace provisioning failed: %v", err)) + return nil } t.updateCondition(condType, privatev1.ConditionStatus_CONDITION_STATUS_TRUE, diff --git a/fulfillment-service/internal/controllers/tenant/tenant_reconciler_function_test.go b/fulfillment-service/internal/controllers/tenant/tenant_reconciler_function_test.go index 49a63387c4..5af7c62682 100644 --- a/fulfillment-service/internal/controllers/tenant/tenant_reconciler_function_test.go +++ b/fulfillment-service/internal/controllers/tenant/tenant_reconciler_function_test.go @@ -2295,6 +2295,125 @@ var _ = Describe("Vault namespace provisioning", func() { Expect(cond.GetStatus()).To(Equal(privatev1.ConditionStatus_CONDITION_STATUS_TRUE)) }) + It("sets FAILED state when vault provisioning fails during initial sync", func() { + reconciler := &function{ + logger: logger, + idpManager: idpManager, + vaultLifecycle: mockVaultClient, + } + + tenant := privatev1.Tenant_builder{ + Id: "org-vault-fail", + Metadata: privatev1.Metadata_builder{ + Name: "vault-fail-org", + Finalizers: []string{finalizers.Controller}, + Tenant: "tenant-1", + }.Build(), + Status: privatev1.TenantStatus_builder{ + BreakGlassCredentials: privatev1.BreakGlassCredentials_builder{ + Username: "vault-fail-org-osac-break-glass", + Password: testPreGeneratedPassword, + }.Build(), + }.Build(), + }.Build() + + mockIDPClient.EXPECT(). + CreateTenant(gomock.Any(), gomock.Any()). + Return(&idp.Tenant{Name: "vault-fail-org", Enabled: true}, nil) + mockIDPClient.EXPECT(). + CreateUser(gomock.Any(), "vault-fail-org", gomock.Any()). + Return(&idp.User{ID: "user-fail"}, nil) + mockIDPClient.EXPECT(). + AssignIdpManagerPermissions(gomock.Any(), "user-fail"). + Return(nil) + + mockVaultClient.EXPECT(). + EnsureTenantNamespace(gomock.Any(), "vault-fail-org"). + Return(fmt.Errorf("vault connection refused")) + + t := &task{r: reconciler, tenant: tenant} + err := t.update(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(tenant.GetStatus().GetState()).To(Equal(privatev1.TenantState_TENANT_STATE_FAILED)) + Expect(tenant.GetStatus().GetMessage()).To(ContainSubstring("Vault namespace provisioning failed")) + Expect(tenant.GetStatus().GetMessage()).To(ContainSubstring("vault connection refused")) + cond := findCondition(tenant) + Expect(cond).ToNot(BeNil()) + Expect(cond.GetStatus()).To(Equal(privatev1.ConditionStatus_CONDITION_STATUS_FALSE)) + Expect(cond.GetReason()).To(Equal("ProvisionFailed")) + }) + + It("sets FAILED state when vault provisioning fails for synced tenant", func() { + reconciler := &function{ + logger: logger, + idpManager: idpManager, + vaultLifecycle: mockVaultClient, + } + + tenant := privatev1.Tenant_builder{ + Id: "org-synced-vault-fail", + Metadata: privatev1.Metadata_builder{ + Name: "synced-vault-fail-org", + Finalizers: []string{finalizers.Controller}, + Tenant: "tenant-1", + }.Build(), + Status: privatev1.TenantStatus_builder{ + State: privatev1.TenantState_TENANT_STATE_SYNCED, + IdpTenantName: "synced-vault-fail-org", + }.Build(), + }.Build() + + mockIDPClient.EXPECT(). + GetTenant(gomock.Any(), "synced-vault-fail-org"). + Return(&idp.Tenant{Name: "synced-vault-fail-org"}, nil) + + mockVaultClient.EXPECT(). + EnsureTenantNamespace(gomock.Any(), "synced-vault-fail-org"). + Return(fmt.Errorf("dial tcp: no such host")) + + t := &task{r: reconciler, tenant: tenant} + err := t.update(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(tenant.GetStatus().GetState()).To(Equal(privatev1.TenantState_TENANT_STATE_FAILED)) + Expect(tenant.GetStatus().GetMessage()).To(ContainSubstring("Vault namespace provisioning failed")) + Expect(tenant.GetStatus().GetMessage()).To(ContainSubstring("dial tcp: no such host")) + cond := findCondition(tenant) + Expect(cond).ToNot(BeNil()) + Expect(cond.GetStatus()).To(Equal(privatev1.ConditionStatus_CONDITION_STATUS_FALSE)) + Expect(cond.GetReason()).To(Equal("ProvisionFailed")) + }) + + It("skips vault provisioning for the system tenant", func() { + reconciler := &function{ + logger: logger, + idpManager: idpManager, + vaultLifecycle: mockVaultClient, + } + + tenant := privatev1.Tenant_builder{ + Id: "org-system", + Metadata: privatev1.Metadata_builder{ + Name: auth.SystemTenant, + Finalizers: []string{finalizers.Controller}, + Tenant: auth.SystemTenant, + }.Build(), + Status: privatev1.TenantStatus_builder{ + State: privatev1.TenantState_TENANT_STATE_SYNCED, + IdpTenantName: auth.SystemTenant, + }.Build(), + }.Build() + + mockIDPClient.EXPECT(). + GetTenant(gomock.Any(), auth.SystemTenant). + Return(&idp.Tenant{Name: auth.SystemTenant}, nil) + + t := &task{r: reconciler, tenant: tenant} + err := t.update(ctx) + Expect(err).ToNot(HaveOccurred()) + cond := findCondition(tenant) + Expect(cond).ToNot(BeNil()) + Expect(cond.GetStatus()).To(Equal(privatev1.ConditionStatus_CONDITION_STATUS_FALSE)) + }) }) var _ = Describe("Vault namespace cleanup during deletion", func() {