OSAC-4909: Reflect vault provisioning failures in tenant status

fulfillment-service/internal/controllers/tenant/tenant_reconciler_function.go

@@ -211,6 +211,9 @@ func (t *task) update(ctx context.Context) error {
 		if err := t.ensureVaultNamespace(ctx); err != nil {
 			return err
 		}
+		if t.tenant.GetStatus().GetState() == privatev1.TenantState_TENANT_STATE_FAILED {
+			return nil
+		}
 		if err := t.ensureDefaultNetworking(ctx); err != nil {
 			return err
 		}
@@ -257,6 +260,9 @@ func (t *task) syncToIDP(ctx context.Context) error {
 	if err := t.ensureVaultNamespace(ctx); err != nil {
 		return err
 	}
+	if t.tenant.GetStatus().GetState() == privatev1.TenantState_TENANT_STATE_FAILED {
+		return nil
+	}
 
 	if err := t.persistBreakGlassSecret(ctx); err != nil {
 		t.r.logger.ErrorContext(ctx, "Failed to persist break-glass credentials secret",
@@ -709,7 +715,11 @@ func (t *task) ensureVaultNamespace(ctx context.Context) error {
 			slog.String("tenant_name", tenantName),
 			slog.Any("error", err),
 		)
-		return fmt.Errorf("failed to provision vault namespace: %w", err)
+		t.updateCondition(condType, privatev1.ConditionStatus_CONDITION_STATUS_FALSE,
+			"ProvisionFailed", fmt.Sprintf("Failed to provision vault namespace: %v", err))
+		t.tenant.GetStatus().SetState(privatev1.TenantState_TENANT_STATE_FAILED)
+		t.tenant.GetStatus().SetMessage(fmt.Sprintf("Vault namespace provisioning failed: %v", err))
+		return nil
 	}
 
 	t.updateCondition(condType, privatev1.ConditionStatus_CONDITION_STATUS_TRUE,

fulfillment-service/internal/controllers/tenant/tenant_reconciler_function_test.go

@@ -2295,6 +2295,125 @@ var _ = Describe("Vault namespace provisioning", func() {
 		Expect(cond.GetStatus()).To(Equal(privatev1.ConditionStatus_CONDITION_STATUS_TRUE))
 	})
 
+	It("sets FAILED state when vault provisioning fails during initial sync", func() {
+		reconciler := &function{
+			logger:         logger,
+			idpManager:     idpManager,
+			vaultLifecycle: mockVaultClient,
+		}
+
+		tenant := privatev1.Tenant_builder{
+			Id: "org-vault-fail",
+			Metadata: privatev1.Metadata_builder{
+				Name:       "vault-fail-org",
+				Finalizers: []string{finalizers.Controller},
+				Tenant:     "tenant-1",
+			}.Build(),
+			Status: privatev1.TenantStatus_builder{
+				BreakGlassCredentials: privatev1.BreakGlassCredentials_builder{
+					Username: "vault-fail-org-osac-break-glass",
+					Password: testPreGeneratedPassword,
+				}.Build(),
+			}.Build(),
+		}.Build()
+
+		mockIDPClient.EXPECT().
+			CreateTenant(gomock.Any(), gomock.Any()).
+			Return(&idp.Tenant{Name: "vault-fail-org", Enabled: true}, nil)
+		mockIDPClient.EXPECT().
+			CreateUser(gomock.Any(), "vault-fail-org", gomock.Any()).
+			Return(&idp.User{ID: "user-fail"}, nil)
+		mockIDPClient.EXPECT().
+			AssignIdpManagerPermissions(gomock.Any(), "user-fail").
+			Return(nil)
+
+		mockVaultClient.EXPECT().
+			EnsureTenantNamespace(gomock.Any(), "vault-fail-org").
+			Return(fmt.Errorf("vault connection refused"))
+
+		t := &task{r: reconciler, tenant: tenant}
+		err := t.update(ctx)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(tenant.GetStatus().GetState()).To(Equal(privatev1.TenantState_TENANT_STATE_FAILED))
+		Expect(tenant.GetStatus().GetMessage()).To(ContainSubstring("Vault namespace provisioning failed"))
+		Expect(tenant.GetStatus().GetMessage()).To(ContainSubstring("vault connection refused"))
+		cond := findCondition(tenant)
+		Expect(cond).ToNot(BeNil())
+		Expect(cond.GetStatus()).To(Equal(privatev1.ConditionStatus_CONDITION_STATUS_FALSE))
+		Expect(cond.GetReason()).To(Equal("ProvisionFailed"))
+	})
+
+	It("sets FAILED state when vault provisioning fails for synced tenant", func() {
+		reconciler := &function{
+			logger:         logger,
+			idpManager:     idpManager,
+			vaultLifecycle: mockVaultClient,
+		}
+
+		tenant := privatev1.Tenant_builder{
+			Id: "org-synced-vault-fail",
+			Metadata: privatev1.Metadata_builder{
+				Name:       "synced-vault-fail-org",
+				Finalizers: []string{finalizers.Controller},
+				Tenant:     "tenant-1",
+			}.Build(),
+			Status: privatev1.TenantStatus_builder{
+				State:         privatev1.TenantState_TENANT_STATE_SYNCED,
+				IdpTenantName: "synced-vault-fail-org",
+			}.Build(),
+		}.Build()
+
+		mockIDPClient.EXPECT().
+			GetTenant(gomock.Any(), "synced-vault-fail-org").
+			Return(&idp.Tenant{Name: "synced-vault-fail-org"}, nil)
+
+		mockVaultClient.EXPECT().
+			EnsureTenantNamespace(gomock.Any(), "synced-vault-fail-org").
+			Return(fmt.Errorf("dial tcp: no such host"))
+
+		t := &task{r: reconciler, tenant: tenant}
+		err := t.update(ctx)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(tenant.GetStatus().GetState()).To(Equal(privatev1.TenantState_TENANT_STATE_FAILED))
+		Expect(tenant.GetStatus().GetMessage()).To(ContainSubstring("Vault namespace provisioning failed"))
+		Expect(tenant.GetStatus().GetMessage()).To(ContainSubstring("dial tcp: no such host"))
+		cond := findCondition(tenant)
+		Expect(cond).ToNot(BeNil())
+		Expect(cond.GetStatus()).To(Equal(privatev1.ConditionStatus_CONDITION_STATUS_FALSE))
+		Expect(cond.GetReason()).To(Equal("ProvisionFailed"))
+	})
+
+	It("skips vault provisioning for the system tenant", func() {
+		reconciler := &function{
+			logger:         logger,
+			idpManager:     idpManager,
+			vaultLifecycle: mockVaultClient,
+		}
+
+		tenant := privatev1.Tenant_builder{
+			Id: "org-system",
+			Metadata: privatev1.Metadata_builder{
+				Name:       auth.SystemTenant,
+				Finalizers: []string{finalizers.Controller},
+				Tenant:     auth.SystemTenant,
+			}.Build(),
+			Status: privatev1.TenantStatus_builder{
+				State:         privatev1.TenantState_TENANT_STATE_SYNCED,
+				IdpTenantName: auth.SystemTenant,
+			}.Build(),
+		}.Build()
+
+		mockIDPClient.EXPECT().
+			GetTenant(gomock.Any(), auth.SystemTenant).
+			Return(&idp.Tenant{Name: auth.SystemTenant}, nil)
+
+		t := &task{r: reconciler, tenant: tenant}
+		err := t.update(ctx)
+		Expect(err).ToNot(HaveOccurred())
+		cond := findCondition(tenant)
+		Expect(cond).ToNot(BeNil())
+		Expect(cond.GetStatus()).To(Equal(privatev1.ConditionStatus_CONDITION_STATUS_FALSE))
+	})
 })
 
 var _ = Describe("Vault namespace cleanup during deletion", func() {