HIVE-29522: Prevent cleaner from creating COMPLETED_COMPACTIONS entries for soft-deleted ACID tables

Author: Venu Reddy

standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/AcidEventListener.java

@@ -19,6 +19,7 @@
 package org.apache.hadoop.hive.metastore;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.AcidConstants;
 import org.apache.hadoop.hive.metastore.api.CompactionRequest;
 import org.apache.hadoop.hive.metastore.api.CompactionType;
 import org.apache.hadoop.hive.metastore.api.Database;
@@ -91,6 +92,7 @@ public void onDropTable(DropTableEvent tableEvent) throws MetaException {
             rqst.setRunas(TxnUtils.findUserToRunAs(table.getSd().getLocation(), table, conf));
             rqst.putToProperties("location", table.getSd().getLocation());
             rqst.putToProperties("ifPurge", Boolean.toString(isMustPurge(tableEvent.getEnvironmentContext(), table)));
+            rqst.putToProperties(AcidConstants.SOFT_DELETE_TABLE, Boolean.TRUE.toString());
             txnHandler.submitForCleanup(rqst, table.getWriteId(), currentTxn);
           } catch (InterruptedException | IOException e) {
             throwMetaException(e);

standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/entities/CompactionInfo.java

@@ -18,6 +18,7 @@
 package org.apache.hadoop.hive.metastore.txn.entities;
 
 import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.hadoop.hive.common.AcidConstants;
 import org.apache.hadoop.hive.common.ValidCompactorWriteIdList;
 import org.apache.hadoop.hive.metastore.api.CompactionInfoStruct;
 import org.apache.hadoop.hive.metastore.api.CompactionType;
@@ -368,4 +369,8 @@ public void setWriteIds(boolean hasUncompactedAborts, Set<Long> writeIds) {
   public boolean isAbortedTxnCleanup() {
     return type == CompactionType.ABORT_TXN_CLEANUP;
   }
+  
+  public boolean isSoftDelete() {
+    return "true".equalsIgnoreCase(getProperty(AcidConstants.SOFT_DELETE_TABLE));
+  }
 }

standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/jdbc/functions/MarkCleanedFunction.java

@@ -50,7 +50,7 @@ public MarkCleanedFunction(CompactionInfo info) {
   public Void execute(MultiDataSourceJdbcResource jdbcResource) throws MetaException {
     NamedParameterJdbcTemplate jdbcTemplate = jdbcResource.getJdbcTemplate();
     MapSqlParameterSource param;
-    if (!info.isAbortedTxnCleanup()) {
+    if (!info.isAbortedTxnCleanup() && !info.isSoftDelete()) {
       param = new MapSqlParameterSource()
           .addValue("id", info.id)
           .addValue("succeeded", Character.toString(SUCCEEDED_STATE), Types.CHAR);
@@ -85,7 +85,7 @@ public Void execute(MultiDataSourceJdbcResource jdbcResource) throws MetaExcepti
      */
     removeCompactionAndAbortRetryEntries(info, jdbcTemplate);
 
-    if (!info.isAbortedTxnCleanup()) {
+    if (!info.isAbortedTxnCleanup() && !info.isSoftDelete()) {
       // Remove entries from completed_txn_components as well, so we don't start looking there
       // again but only up to the highest write ID include in this compaction job.
       //highestWriteId will be NULL in upgrade scenarios
@@ -121,7 +121,9 @@ public Void execute(MultiDataSourceJdbcResource jdbcResource) throws MetaExcepti
     }
 
     // Do cleanup of metadata in TXN_COMPONENTS table.
-    removeTxnComponents(info, jdbcResource);
+    if (!info.isSoftDelete()) {
+      removeTxnComponents(info, jdbcResource);
+    }
     return null;
   }
 
@@ -175,7 +177,7 @@ private void removeCompactionAndAbortRetryEntries(CompactionInfo info, NamedPara
     String deleteQuery = """
         DELETE FROM "COMPACTION_QUEUE" WHERE "CQ_ID" = :id
         """;
-    if (!info.isAbortedTxnCleanup()) {
+    if (!info.isAbortedTxnCleanup() && !info.isSoftDelete()) {
       deleteQuery += """
           OR ("CQ_DATABASE" = :db AND "CQ_TABLE" = :table
             AND (:partition is NULL OR "CQ_PARTITION" = :partition)