Fixes 4065: TestCase reindex preserves testCaseStatus + lenient time-series reads

openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/DistributedReindexFinalizer.java

@@ -125,13 +125,15 @@ private void promoteColumnIndex(boolean tableSuccess, Set<String> finalizedEntit
 
   private boolean computeEntitySuccess(
       String entityType, Map<String, SearchIndexJob.EntityTypeStats> entityStats) {
-    if (entityStats == null || entityStats.isEmpty()) {
-      return false;
+    if (entityStats == null || entityStats.isEmpty() || entityStats.get(entityType) == null) {
+      // No stats recorded for this entity type means the reader did zero work — either
+      // the source had 0 rows, or the entity is driven by a parallel pipeline (e.g.
+      // vectorEmbedding via RecreateWithEmbeddings) that doesn't feed the reader→sink
+      // stats. Treat absent stats as success so the staged index gets promoted rather
+      // than the job rolling up to FAILED on an entity that has nothing to fail on.
+      return true;
     }
     SearchIndexJob.EntityTypeStats stats = entityStats.get(entityType);
-    if (stats == null) {
-      return false;
-    }
     return stats.getFailedRecords() == 0
         && stats.getSuccessRecords() + stats.getFailedRecords() >= stats.getTotalRecords();
   }

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityTimeSeriesRepository.java

@@ -463,7 +463,8 @@ public ResultList<T> listFromSearchWithOffset(
               searchListFilter, limit, offset, entityType, searchSortFilter, q, queryString);
       total = results.getTotal();
       for (Map<String, Object> json : results.getResults()) {
-        T entity = setFieldsInternal(JsonUtils.readOrConvertValue(json, entityClass), fields);
+        T entity =
+            setFieldsInternal(JsonUtils.readOrConvertValueLenient(json, entityClass), fields);
         try {
           setInheritedFields(entity);
         } catch (RuntimeException e) {
@@ -526,7 +527,7 @@ public ResultList<T> listLatestFromSearch(
                     Map<String, Object> source = extractAndFilterSource(hit);
                     T entity =
                         setFieldsInternal(
-                            JsonUtils.readOrConvertValue(source, entityClass), fields);
+                            JsonUtils.readOrConvertValueLenient(source, entityClass), fields);
                     if (entity != null) {
                       try {
                         setInheritedFields(entity);
@@ -684,7 +685,7 @@ public T latestFromSearch(EntityUtil.Fields fields, SearchListFilter searchListF
     SearchResultListMapper results =
         searchRepository.listWithOffset(searchListFilter, 1, 0, entityType, searchSortFilter, q);
     for (Map<String, Object> json : results.getResults()) {
-      T entity = setFieldsInternal(JsonUtils.readOrConvertValue(json, entityClass), fields);
+      T entity = setFieldsInternal(JsonUtils.readOrConvertValueLenient(json, entityClass), fields);
       setInheritedFields(entity);
       clearFieldsInternal(entity, fields);
       return entity;

openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/APIEndpointIndex.java

@@ -32,6 +32,16 @@ public String getEntityTypeName() {
     return Entity.API_ENDPOINT;
   }
 
+  @Override
+  public Set<String> getRequiredReindexFields() {
+    Set<String> fields = new java.util.HashSet<>(DataAssetIndex.super.getRequiredReindexFields());
+    // APIEndpointRepository.fetchAndSetSchemaFieldTagsInBatch is gated on
+    // fields.contains("requestSchema") || fields.contains("responseSchema").
+    fields.add("requestSchema");
+    fields.add("responseSchema");
+    return java.util.Collections.unmodifiableSet(fields);
+  }
+
   @Override
   public Set<String> getExcludedFields() {
     return excludeAPIEndpointFields;

openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TaggableIndex.java

@@ -37,8 +37,9 @@ default void applyTagFields(Map<String, Object> doc) {
     if (!(entity instanceof EntityInterface ei)) {
       return;
     }
-    ParseTags parseTags = new ParseTags(Entity.getEntityTags(getEntityTypeName(), ei));
-    doc.put("tags", parseTags.getTags());
+    List<TagLabel> entityTags = Entity.getEntityTags(getEntityTypeName(), ei);
+    ParseTags parseTags = new ParseTags(entityTags);
+    doc.put("tags", entityTags);
     doc.put("tier", parseTags.getTierTag());
     doc.put("classificationTags", parseTags.getClassificationTags());
     doc.put("glossaryTags", parseTags.getGlossaryTags());

openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseIndex.java

@@ -37,6 +37,8 @@ public Set<String> getRequiredReindexFields() {
     fields.add("testSuite");
     fields.add("testSuites");
     fields.add("testDefinition");
+    fields.add(Entity.TEST_CASE_RESULT);
+    fields.add("incidentId");
     return java.util.Collections.unmodifiableSet(fields);
   }
 

openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestSuiteIndex.java

@@ -1,5 +1,7 @@
 package org.openmetadata.service.search.indexes;
 
+import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -29,6 +31,13 @@ public Set<String> getExcludedFields() {
     return excludeFields;
   }
 
+  @Override
+  public Set<String> getRequiredReindexFields() {
+    Set<String> fields = new HashSet<>(TaggableIndex.super.getRequiredReindexFields());
+    fields.add("summary");
+    return Collections.unmodifiableSet(fields);
+  }
+
   public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
     setParentRelationships(doc, testSuite);
 

openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TopicIndex.java

@@ -44,6 +44,15 @@ public Object getIndexServiceType() {
     return topic.getServiceType();
   }
 
+  @Override
+  public Set<String> getRequiredReindexFields() {
+    Set<String> fields = new java.util.HashSet<>(DataAssetIndex.super.getRequiredReindexFields());
+    // TopicRepository.bulkPopulateEntityFieldTags only fires when fields.contains("messageSchema");
+    // without it nested schema-field tags are not hydrated and _source.tags loses the merge.
+    fields.add("messageSchema");
+    return java.util.Collections.unmodifiableSet(fields);
+  }
+
   public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
     if (topic.getMessageSchema() != null
         && topic.getMessageSchema().getSchemaFields() != null

openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/WorksheetIndex.java

@@ -25,6 +25,15 @@ public Object getEntity() {
     return worksheet;
   }
 
+  @Override
+  public Set<String> getRequiredReindexFields() {
+    Set<String> fields = new java.util.HashSet<>(DataAssetIndex.super.getRequiredReindexFields());
+    // WorksheetRepository.clearFields actively nulls columns when not requested;
+    // also column-level tag hydration in setFields is gated on fields.contains("columns").
+    fields.add("columns");
+    return java.util.Collections.unmodifiableSet(fields);
+  }
+
   @Override
   public String getEntityTypeName() {
     return Entity.WORKSHEET;

openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/DistributedIndexingStrategyTest.java

@@ -414,7 +414,8 @@ void finalizeAllEntityReindexPromotesZeroRecordEntityFromInitializedStats() thro
   }
 
   @Test
-  void finalizeAllEntityReindexSkipsPromotedEntitiesAndFailsMissingEntityStats() throws Exception {
+  void finalizeAllEntityReindexSkipsPromotedEntitiesAndPromotesMissingEntityStats()
+      throws Exception {
     DistributedSearchIndexExecutor executor = mock(DistributedSearchIndexExecutor.class);
     EntityCompletionTracker tracker = mock(EntityCompletionTracker.class);
     RecreateIndexHandler indexPromotionHandler = mock(RecreateIndexHandler.class);
@@ -478,7 +479,13 @@ void finalizeAllEntityReindexSkipsPromotedEntitiesAndFailsMissingEntityStats() t
           contextCaptor.getAllValues().get(i).getEntityType(), successCaptor.getAllValues().get(i));
     }
 
-    assertEquals(Boolean.FALSE, outcomes.get("user"));
+    // 084c5c2205 flipped the contract: an entity with no stats recorded means the reader
+    // did zero work (source had 0 rows, or the entity is driven by a parallel pipeline like
+    // vectorEmbedding/RecreateWithEmbeddings). Such entities are now promoted as success so
+    // the staged index gets swapped in instead of the job rolling up to FAILED on something
+    // that had nothing to fail on. `dashboard` still fails because its stats record a real
+    // failure (failedRecords=1).
+    assertEquals(Boolean.TRUE, outcomes.get("user"));
     assertEquals(Boolean.FALSE, outcomes.get("dashboard"));
   }
 

openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/DistributedReindexFinalizerTest.java

@@ -39,6 +39,29 @@ void finalizeRemainingEntitiesPromotesColumnOnceWhenTableAndColumnRemain() {
     assertEquals(Boolean.TRUE, finalizations.get(Entity.TABLE_COLUMN));
   }
 
+  @Test
+  void finalizeRemainingEntitiesTreatsEntityWithoutStatsAsSuccess() {
+    // vectorEmbedding has a staged index (built by RecreateWithEmbeddings) but the
+    // distributed reader/sink pipeline doesn't run for it, so entityStats has no entry.
+    // The finalizer must still promote the staged index rather than mark the entity
+    // failed and roll the whole job up to FAILED.
+    RecreateIndexHandler indexPromotionHandler = mock(RecreateIndexHandler.class);
+    ReindexContext stagedIndexContext = stagedContext("vectorEmbedding");
+
+    DistributedReindexFinalizer finalizer =
+        new DistributedReindexFinalizer(indexPromotionHandler, stagedIndexContext);
+    finalizer.finalizeRemainingEntities(Set.of(), Map.of(), true);
+
+    ArgumentCaptor<EntityReindexContext> contextCaptor =
+        ArgumentCaptor.forClass(EntityReindexContext.class);
+    ArgumentCaptor<Boolean> successCaptor = ArgumentCaptor.forClass(Boolean.class);
+    verify(indexPromotionHandler, times(1))
+        .finalizeReindex(contextCaptor.capture(), successCaptor.capture());
+
+    assertEquals("vectorEmbedding", contextCaptor.getValue().getEntityType());
+    assertEquals(Boolean.TRUE, successCaptor.getValue());
+  }
+
   @Test
   void finalizeRemainingEntitiesDoesNotRepromoteAlreadyPromotedColumnWhenTableRemains() {
     RecreateIndexHandler indexPromotionHandler = mock(RecreateIndexHandler.class);

openmetadata-service/src/test/java/org/openmetadata/service/search/SearchIndexFactoryTest.java

@@ -209,6 +209,17 @@ void reindexFieldsIncludeKnownOverrides() {
     assertTrue(testCaseFields.contains("testSuite"));
     assertTrue(testCaseFields.contains("testSuites"));
     assertTrue(testCaseFields.contains("testDefinition"));
+    // Regression: testCaseResult/incidentId are stripped from storage JSON and
+    // only fetched by setFieldsInBulk when explicitly requested. Reindex without
+    // them produces docs missing testCaseStatus, blanking statuses in the UI.
+    assertTrue(testCaseFields.contains(Entity.TEST_CASE_RESULT));
+    assertTrue(testCaseFields.contains("incidentId"));
+    // TestSuiteRepository registers a fetcher for "summary" that populates
+    // testCaseResultSummary. The DQ TestSuites list page sorts by the
+    // top-level lastResultTimestamp field (computed in TestSuiteIndex from
+    // that summary) and renders a success-% column per row. Without
+    // "summary" the fetcher never runs and the ES doc has neither field.
+    assertTrue(factory.getReindexFieldsFor(Entity.TEST_SUITE).contains("summary"));
   }
 
   @Test

openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/TestCaseIndexTest.java

@@ -15,6 +15,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.UUID;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
@@ -144,6 +145,28 @@ void testBuildSearchIndexDoc_endToEnd_hasCommonAndTagFields() {
     assertNotNull(result.get("originEntityFQN"));
   }
 
+  @Test
+  void testRequiredReindexFields_includesTestCaseResultAndIncidentId() {
+    // Regression test for the 1.12.7 reindex bug: testCaseResult and incidentId
+    // are stripped from the storage JSON and only loaded by
+    // TestCaseRepository.setFieldsInBulk when present in the requested field
+    // set. If they are not in getRequiredReindexFields(), the reindexer writes
+    // a doc with no testCaseStatus and the UI/search shows test cases with no
+    // status until a per-case write re-populates them.
+    TestCase tc = new TestCase().withId(UUID.randomUUID()).withName("tc");
+    Set<String> required = new TestCaseIndex(tc).getRequiredReindexFields();
+
+    assertTrue(
+        required.contains(Entity.TEST_CASE_RESULT),
+        "TestCaseIndex.getRequiredReindexFields() must include 'testCaseResult'");
+    assertTrue(
+        required.contains("incidentId"),
+        "TestCaseIndex.getRequiredReindexFields() must include 'incidentId'");
+    assertTrue(required.contains("testSuite"));
+    assertTrue(required.contains("testSuites"));
+    assertTrue(required.contains("testDefinition"));
+  }
+
   @Test
   void testBuildSearchIndexDocInternal_testDefinitionNotFound() {
     UUID testDefId = UUID.randomUUID();

openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/DataQuality/TestCaseStatusAfterReindex.spec.ts

@@ -0,0 +1,122 @@
+/*
+ *  Copyright 2026 Collate.
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/**
+ * Regression for the 1.12.7 selective-reindex bug
+ * (https://github.com/open-metadata/OpenMetadata/pull/27723):
+ * TestCaseIndex.getRequiredReindexFields() omitted `testCaseResult` and
+ * `incidentId`, both of which are stripped from the storage JSON. On reindex,
+ * TestCaseRepository.setFieldsInBulk skipped fetching them and the resulting
+ * ES doc had no `testCaseStatus` — wiping status from search/UI until a
+ * per-case write re-populated it.
+ *
+ * This test creates a test case, writes a result, forces an entity reindex
+ * with `recreate=true` (delete + re-add), and asserts the status survives.
+ */
+
+import test, { expect } from '@playwright/test';
+import { TableClass } from '../../../support/entity/TableClass';
+import { createNewPage } from '../../../utils/common';
+
+test.use({ storageState: 'playwright/.auth/admin.json' });
+
+const TEST_CASE_STATUS = 'Failed' as const;
+
+test('Test case status survives a full entity reindex', async ({ browser }) => {
+  const { apiContext, afterAction } = await createNewPage(browser);
+
+  const table = new TableClass();
+
+  try {
+    await table.create(apiContext);
+
+    const testCase = await table.createTestCase(apiContext);
+    const testCaseFqn = testCase.fullyQualifiedName as string;
+    const testCaseId = testCase.id as string;
+
+    await table.addTestCaseResult(apiContext, testCaseFqn, {
+      result: 'Reindex regression check',
+      testCaseStatus: TEST_CASE_STATUS,
+      timestamp: Date.now(),
+    });
+
+    // Wait for the search doc to settle and assert the status is indexed.
+    await expect
+      .poll(
+        async () => {
+          const res = await apiContext.get(
+            `/api/v1/search/query?q=fullyQualifiedName:%22${encodeURIComponent(
+              testCaseFqn
+            )}%22&index=test_case_search_index`
+          );
+          if (res.status() !== 200) {
+            return undefined;
+          }
+          const body = await res.json();
+          const hits = body?.hits?.hits ?? [];
+          return hits[0]?._source?.testCaseResult?.testCaseStatus;
+        },
+        {
+          message:
+            'pre-reindex: test case search doc must include testCaseResult.testCaseStatus',
+          timeout: 30_000,
+        }
+      )
+      .toBe(TEST_CASE_STATUS);
+
+    // Force a recreate-style reindex of the test case — this is the exact
+    // path that drops the status before the fix.
+    const reindexRes = await apiContext.post(
+      '/api/v1/search/reindexEntities?recreate=true',
+      {
+        data: [
+          {
+            id: testCaseId,
+            type: 'testCase',
+            fullyQualifiedName: testCaseFqn,
+          },
+        ],
+      }
+    );
+
+    expect(reindexRes.status()).toBeLessThan(400);
+
+    // Assert the status is still there after reindex. Before the fix, the
+    // recreated doc had no testCaseResult and this poll would time out.
+    await expect
+      .poll(
+        async () => {
+          const res = await apiContext.get(
+            `/api/v1/search/query?q=fullyQualifiedName:%22${encodeURIComponent(
+              testCaseFqn
+            )}%22&index=test_case_search_index`
+          );
+          if (res.status() !== 200) {
+            return undefined;
+          }
+          const body = await res.json();
+          const hits = body?.hits?.hits ?? [];
+          return hits[0]?._source?.testCaseResult?.testCaseStatus;
+        },
+        {
+          message:
+            'post-reindex: test case search doc must still include testCaseResult.testCaseStatus',
+          timeout: 30_000,
+        }
+      )
+      .toBe(TEST_CASE_STATUS);
+  } finally {
+    await table.delete(apiContext);
+    await afterAction();
+  }
+});