From c7281baa592bca2256330bc3713b7a08afd5c65f Mon Sep 17 00:00:00 2001 From: Seth Date: Mon, 5 Jan 2026 13:38:48 -0500 Subject: [PATCH 01/16] Remove interconnected IMA, Thrift, and ClientImpl non-public methods - Remove extends ClientContext from InMemoryAccumuloClient - Remove extends Connector from InMemoryConnector - Remove implements Instance from InMemoryInstance - Delete InMemoryClientInfo.java (used non-public Credentials/ClientInfoImpl) - Clean up BulkInputFormat TabletLocator usage - Update PushdownScheduler to remove ClientContext usage Part of #2443 --- .../inmemory/InMemoryAccumuloClient.java | 46 +++++++++-------- .../accumulo/inmemory/InMemoryClientInfo.java | 23 --------- .../accumulo/inmemory/InMemoryConnector.java | 51 ++++--------------- .../accumulo/inmemory/InMemoryInstance.java | 46 +---------------- .../datawave/mr/bulk/BulkInputFormat.java | 20 +++----- .../query/scheduler/PushdownScheduler.java | 11 +--- 6 files changed, 46 insertions(+), 151 deletions(-) delete mode 100644 core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryClientInfo.java diff --git a/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryAccumuloClient.java b/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryAccumuloClient.java index bcb097e396e..75d56a29bb5 100644 --- a/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryAccumuloClient.java +++ b/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryAccumuloClient.java @@ -30,35 +30,31 @@ import org.apache.accumulo.core.client.MultiTableBatchWriter; import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.client.TableNotFoundException; + import org.apache.accumulo.core.client.admin.InstanceOperations; import org.apache.accumulo.core.client.admin.NamespaceOperations; import org.apache.accumulo.core.client.admin.ReplicationOperations; import org.apache.accumulo.core.client.admin.SecurityOperations; import org.apache.accumulo.core.client.admin.TableOperations; import org.apache.accumulo.core.client.security.tokens.PasswordToken; -import org.apache.accumulo.core.clientImpl.ClientContext; -import org.apache.accumulo.core.clientImpl.Credentials; -import org.apache.accumulo.core.clientImpl.thrift.SecurityErrorCode; -import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.SystemPermission; -import org.apache.accumulo.core.singletons.SingletonReservation; -public class InMemoryAccumuloClient extends ClientContext implements AccumuloClient { +// Remove `extends ClientContext` since it's deprecated. +// Need a workaround for the features that are no longer supported. Not sure if we can just throw them away atm. + +public class InMemoryAccumuloClient implements AccumuloClient { String username; private final InMemoryAccumulo acu; + private ConditionalWriterConfig conditionalWriterConfig; + private volatile boolean closed = false; + public InMemoryAccumuloClient(String username, InMemoryInstance instance) throws AccumuloSecurityException { - this(new Credentials(username, new PasswordToken(new byte[0])), instance.acu); - } - public InMemoryAccumuloClient(Credentials credentials, InMemoryAccumulo acu) throws AccumuloSecurityException { - super(SingletonReservation.noop(), new InMemoryClientInfo(credentials), DefaultConfiguration.getInstance(), null); - if (credentials.getToken().isDestroyed()) - throw new AccumuloSecurityException(credentials.getPrincipal(), SecurityErrorCode.TOKEN_EXPIRED); - this.username = credentials.getPrincipal(); - this.acu = acu; + this.username = username; + this.acu = instance.acu; if (!acu.users.containsKey(username)) { InMemoryUser user = new InMemoryUser(username, new PasswordToken(new byte[0]), Authorizations.EMPTY); user.permissions.add(SystemPermission.SYSTEM); @@ -66,6 +62,22 @@ public InMemoryAccumuloClient(Credentials credentials, InMemoryAccumulo acu) thr } } + @Override + public ConditionalWriter createConditionalWriter(String tableName, ConditionalWriterConfig config) throws TableNotFoundException { + // TODO add implementation + throw new UnsupportedOperationException(); + } + + public ConditionalWriter createConditionalWriter(String tableName) throws TableNotFoundException { + return this.createConditionalWriter(tableName, (ConditionalWriterConfig)null); + } + + private void ensureOpen() { + if (this.closed) { + throw new IllegalStateException("This client was closed."); + } + } + @Override public BatchScanner createBatchScanner(String tableName, Authorizations authorizations, int numQueryThreads) throws TableNotFoundException { if (acu.tables.get(tableName) == null) @@ -156,12 +168,6 @@ public NamespaceOperations namespaceOperations() { return new InMemoryNamespaceOperations(acu, username); } - @Override - public ConditionalWriter createConditionalWriter(String tableName, ConditionalWriterConfig config) { - // TODO add implementation - throw new UnsupportedOperationException(); - } - @Override public ReplicationOperations replicationOperations() { // TODO add implementation diff --git a/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryClientInfo.java b/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryClientInfo.java deleted file mode 100644 index 4f172a55401..00000000000 --- a/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryClientInfo.java +++ /dev/null @@ -1,23 +0,0 @@ -package datawave.accumulo.inmemory; - -import java.util.Properties; - -import org.apache.accumulo.core.client.security.tokens.PasswordToken; -import org.apache.accumulo.core.clientImpl.ClientInfoImpl; -import org.apache.accumulo.core.clientImpl.Credentials; -import org.apache.accumulo.core.conf.ClientProperty; - -public class InMemoryClientInfo extends ClientInfoImpl { - public InMemoryClientInfo(Credentials credentials) { - super(toProperties(credentials), credentials.getToken()); - } - - private static Properties toProperties(Credentials credentials) { - Properties props = new Properties(); - props.put(ClientProperty.INSTANCE_NAME.getKey(), new InMemoryInstance().instanceName); - props.put(ClientProperty.AUTH_PRINCIPAL.getKey(), credentials.getPrincipal()); - props.put(ClientProperty.AUTH_TOKEN.getKey(), new String(((PasswordToken) (credentials.getToken())).getPassword())); - props.put(ClientProperty.AUTH_TYPE.getKey(), "password"); - return props; - } -} diff --git a/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryConnector.java b/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryConnector.java index 5708d0cfa84..37164b070a6 100644 --- a/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryConnector.java +++ b/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryConnector.java @@ -25,47 +25,39 @@ import org.apache.accumulo.core.client.BatchWriterConfig; import org.apache.accumulo.core.client.ConditionalWriter; import org.apache.accumulo.core.client.ConditionalWriterConfig; -import org.apache.accumulo.core.client.Connector; -import org.apache.accumulo.core.client.Instance; import org.apache.accumulo.core.client.MultiTableBatchWriter; import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.admin.InstanceOperations; import org.apache.accumulo.core.client.admin.NamespaceOperations; -import org.apache.accumulo.core.client.admin.ReplicationOperations; import org.apache.accumulo.core.client.admin.SecurityOperations; import org.apache.accumulo.core.client.admin.TableOperations; import org.apache.accumulo.core.client.security.tokens.PasswordToken; -import org.apache.accumulo.core.clientImpl.Credentials; -import org.apache.accumulo.core.clientImpl.thrift.SecurityErrorCode; import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.core.security.SystemPermission; -public class InMemoryConnector extends Connector { +public class InMemoryConnector { String username; private final InMemoryAccumulo acu; - private final Instance instance; - InMemoryConnector(String username, InMemoryInstance instance) throws AccumuloSecurityException { - this(new Credentials(username, new PasswordToken(new byte[0])), new InMemoryAccumulo(InMemoryInstance.getDefaultFileSystem()), instance); - } + InMemoryConnector(String username, InMemoryInstance instance) throws AccumuloSecurityException { - InMemoryConnector(Credentials credentials, InMemoryAccumulo acu, InMemoryInstance instance) throws AccumuloSecurityException { - if (credentials.getToken().isDestroyed()) - throw new AccumuloSecurityException(credentials.getPrincipal(), SecurityErrorCode.TOKEN_EXPIRED); - this.username = credentials.getPrincipal(); - this.acu = acu; - this.instance = instance; + this.username = username; + this.acu = instance.acu; + if (!acu.users.containsKey(username)) { + InMemoryUser user = new InMemoryUser(username, new PasswordToken(new byte[0]), Authorizations.EMPTY); + user.permissions.add(SystemPermission.SYSTEM); + acu.users.put(user.name, user); + } } - @Override public BatchScanner createBatchScanner(String tableName, Authorizations authorizations, int numQueryThreads) throws TableNotFoundException { if (acu.tables.get(tableName) == null) throw new TableNotFoundException(tableName, tableName, "no such table"); return acu.createBatchScanner(tableName, authorizations); } - @Override public BatchDeleter createBatchDeleter(String tableName, Authorizations authorizations, int numQueryThreads, long maxMemory, long maxLatency, int maxWriteThreads) throws TableNotFoundException { if (acu.tables.get(tableName) == null) @@ -73,36 +65,30 @@ public BatchDeleter createBatchDeleter(String tableName, Authorizations authoriz return new InMemoryBatchDeleter(acu, tableName, authorizations); } - @Override public BatchDeleter createBatchDeleter(String tableName, Authorizations authorizations, int numQueryThreads, BatchWriterConfig config) throws TableNotFoundException { return createBatchDeleter(tableName, authorizations, numQueryThreads, config.getMaxMemory(), config.getMaxLatency(TimeUnit.MILLISECONDS), config.getMaxWriteThreads()); } - @Override public BatchWriter createBatchWriter(String tableName, long maxMemory, long maxLatency, int maxWriteThreads) throws TableNotFoundException { if (acu.tables.get(tableName) == null) throw new TableNotFoundException(tableName, tableName, "no such table"); return new InMemoryBatchWriter(acu, tableName); } - @Override public BatchWriter createBatchWriter(String tableName, BatchWriterConfig config) throws TableNotFoundException { return createBatchWriter(tableName, config.getMaxMemory(), config.getMaxLatency(TimeUnit.MILLISECONDS), config.getMaxWriteThreads()); } - @Override public MultiTableBatchWriter createMultiTableBatchWriter(long maxMemory, long maxLatency, int maxWriteThreads) { return new InMemoryMultiTableBatchWriter(acu); } - @Override public MultiTableBatchWriter createMultiTableBatchWriter(BatchWriterConfig config) { return createMultiTableBatchWriter(config.getMaxMemory(), config.getMaxLatency(TimeUnit.MILLISECONDS), config.getMaxWriteThreads()); } - @Override public Scanner createScanner(String tableName, Authorizations authorizations) throws TableNotFoundException { InMemoryTable table = acu.tables.get(tableName); if (table == null) @@ -110,46 +96,29 @@ public Scanner createScanner(String tableName, Authorizations authorizations) th return new InMemoryScanner(table, authorizations); } - @Override - public Instance getInstance() { - return instance; - } - - @Override public String whoami() { return username; } - @Override public TableOperations tableOperations() { return new InMemoryTableOperations(acu, username); } - @Override public SecurityOperations securityOperations() { return new InMemorySecurityOperations(acu); } - @Override public InstanceOperations instanceOperations() { return new InMemoryInstanceOperations(acu); } - @Override public NamespaceOperations namespaceOperations() { return new InMemoryNamespaceOperations(acu, username); } - @Override public ConditionalWriter createConditionalWriter(String tableName, ConditionalWriterConfig config) throws TableNotFoundException { // TODO add implementation throw new UnsupportedOperationException(); } - @Override - public ReplicationOperations replicationOperations() { - // TODO add implementation - throw new UnsupportedOperationException(); - } - } diff --git a/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryInstance.java b/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryInstance.java index 62589f01c54..9e0fd620ee7 100644 --- a/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryInstance.java +++ b/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryInstance.java @@ -17,25 +17,14 @@ package datawave.accumulo.inmemory; import java.io.IOException; -import java.nio.ByteBuffer; import java.util.Collections; import java.util.HashMap; + import java.util.List; import java.util.Map; -import org.apache.accumulo.core.client.AccumuloException; -import org.apache.accumulo.core.client.AccumuloSecurityException; -import org.apache.accumulo.core.client.Connector; -import org.apache.accumulo.core.client.Instance; -import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; -import org.apache.accumulo.core.client.security.tokens.PasswordToken; -import org.apache.accumulo.core.clientImpl.Credentials; -import org.apache.accumulo.core.clientImpl.thrift.SecurityErrorCode; -import org.apache.accumulo.core.util.ByteBufferUtil; -import org.apache.accumulo.core.util.TextUtil; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.io.Text; /** * InMemory Accumulo provides an in memory implementation of the Accumulo client API. It is possible that the behavior of this implementation may differ subtly @@ -48,7 +37,7 @@ * Accumulo. * */ -public class InMemoryInstance implements Instance { +public class InMemoryInstance { static final String genericAddress = "localhost:1234"; static final Map instances = new HashMap<>(); @@ -85,61 +74,30 @@ public InMemoryInstance(String instanceName, FileSystem fs) { this.instanceName = instanceName; } - @Override public String getRootTabletLocation() { return genericAddress; } - @Override public List getMasterLocations() { return Collections.singletonList(genericAddress); } - @Override public String getInstanceID() { return "mock-instance-id"; } - @Override public String getInstanceName() { return instanceName; } - @Override public String getZooKeepers() { return "localhost"; } - @Override public int getZooKeepersSessionTimeOut() { return 30 * 1000; } - @Override - public Connector getConnector(String user, byte[] pass) throws AccumuloException, AccumuloSecurityException { - return getConnector(user, new PasswordToken(pass)); - } - - @Override - public Connector getConnector(String user, ByteBuffer pass) throws AccumuloException, AccumuloSecurityException { - return getConnector(user, ByteBufferUtil.toBytes(pass)); - } - - @Override - public Connector getConnector(String user, CharSequence pass) throws AccumuloException, AccumuloSecurityException { - return getConnector(user, TextUtil.getBytes(new Text(pass.toString()))); - } - - @Override - public Connector getConnector(String principal, AuthenticationToken token) throws AccumuloException, AccumuloSecurityException { - Connector conn = new InMemoryConnector(new Credentials(principal, token), acu, this); - if (!acu.users.containsKey(principal)) - conn.securityOperations().createLocalUser(principal, (PasswordToken) token); - else if (!acu.users.get(principal).token.equals(token)) - throw new AccumuloSecurityException(principal, SecurityErrorCode.BAD_CREDENTIALS); - return conn; - } - public static class CachedConfiguration { private static Configuration configuration = null; diff --git a/warehouse/core/src/main/java/datawave/mr/bulk/BulkInputFormat.java b/warehouse/core/src/main/java/datawave/mr/bulk/BulkInputFormat.java index 7b70c238a7a..bdc6513228f 100644 --- a/warehouse/core/src/main/java/datawave/mr/bulk/BulkInputFormat.java +++ b/warehouse/core/src/main/java/datawave/mr/bulk/BulkInputFormat.java @@ -37,9 +37,6 @@ import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.TableOfflineException; import org.apache.accumulo.core.client.security.tokens.PasswordToken; -import org.apache.accumulo.core.clientImpl.ClientConfConverter; -import org.apache.accumulo.core.clientImpl.ClientContext; -import org.apache.accumulo.core.clientImpl.ClientInfo; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.PartialKey; import org.apache.accumulo.core.data.Range; @@ -49,15 +46,12 @@ import org.apache.accumulo.core.iterators.SortedKeyValueIterator; import org.apache.accumulo.core.iterators.user.RegExFilter; import org.apache.accumulo.core.iterators.user.VersioningIterator; -import org.apache.accumulo.core.manager.state.tables.TableState; import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.schema.MetadataSchema; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.accumulo.core.security.TablePermission; -import org.apache.accumulo.core.singletons.SingletonManager; import org.apache.accumulo.core.util.format.DateFormatSupplier; -import org.apache.accumulo.core.util.threads.Threads; import org.apache.commons.codec.binary.Base64; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.conf.Configuration; @@ -1045,7 +1039,7 @@ protected static LocationStrategy getLocationStrategy(Configuration conf) { return new DefaultLocationStrategy(); } - public List binRanges(ClientContext context, List ranges, Map>> binnedRanges) + public List binRanges(List ranges, Map>> binnedRanges) throws AccumuloException, AccumuloSecurityException, TableNotFoundException { binnedRanges.put("", Collections.singletonMap(new KeyExtent(TableId.of(""), null, null), ranges)); return Collections.emptyList(); @@ -1083,16 +1077,14 @@ public List getSplits(JobContext job) throws IOException { } else { try (AccumuloClient client = getClient(job.getConfiguration())) { TableId tableId = null; - ClientInfo info = ClientInfo.from(cbHelper.newClientProperties()); - ClientContext context = new ClientContext(SingletonManager.getClientReservation(), info, - ClientConfConverter.toAccumuloConf(info.getProperties()), Threads.UEH); - while (!binRanges(context, ranges, binnedRanges).isEmpty()) { + + while (!binRanges(ranges, binnedRanges).isEmpty()) { if (!(client instanceof InMemoryAccumuloClient)) { if (tableId == null) - tableId = context.getTableId(tableName); - if (!context.tableNodeExists(tableId)) + tableId = TableId.of(client.tableOperations().tableIdMap().get(tableName));// TABLE ID todo:ensure this is correct + if (!client.tableOperations().tableIdMap().containsKey(tableName))// CHECK IF TABLE ID EXISTS //todo same throw new TableDeletedException(tableId.canonical()); - if (context.getTableState(tableId) == TableState.OFFLINE) + if (!client.tableOperations().isOnline(tableName)) // CHECK IF TABLE ID OFFLInE //todo same throw new TableOfflineException("Table (" + tableId.canonical() + ") is offline"); } binnedRanges.clear(); diff --git a/warehouse/query-core/src/main/java/datawave/query/scheduler/PushdownScheduler.java b/warehouse/query-core/src/main/java/datawave/query/scheduler/PushdownScheduler.java index c6a393cfeff..284576ce5a1 100644 --- a/warehouse/query-core/src/main/java/datawave/query/scheduler/PushdownScheduler.java +++ b/warehouse/query-core/src/main/java/datawave/query/scheduler/PushdownScheduler.java @@ -14,7 +14,6 @@ import org.apache.accumulo.core.client.BatchScanner; import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.client.TableNotFoundException; -import org.apache.accumulo.core.clientImpl.ClientContext; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.data.Value; @@ -27,8 +26,6 @@ import com.google.common.collect.Iterators; import com.google.common.collect.Lists; -import datawave.accumulo.inmemory.InMemoryAccumuloClient; -import datawave.core.common.connection.AccumuloConnectionFactory; import datawave.core.common.logging.ThreadConfigurableLogger; import datawave.core.query.configuration.QueryData; import datawave.core.query.configuration.Result; @@ -153,12 +150,8 @@ protected Iterator concatIterators() throws AccumuloException, AccumuloS Set auths = config.getAuthorizations(); AccumuloClient client = config.getClient(); - if (client instanceof InMemoryAccumuloClient) { - tableId = TableId.of(config.getTableName()); - } else { - ClientContext ctx = AccumuloConnectionFactory.getClientContext(client); - tableId = ctx.getTableId(tableName); - } + String tableIdStr = client.tableOperations().tableIdMap().get(tableName); + tableId = TableId.of(tableIdStr); Iterator> chunkIter = Iterators.transform(getQueryDataIterator(), getPushdownFunction()); From bb272c5ab688fd7ad97f2779dbc54b6e971c940e Mon Sep 17 00:00:00 2001 From: Seth Date: Tue, 20 Jan 2026 11:04:22 -0500 Subject: [PATCH 02/16] Fix code formatting issues Apply formatter, sortpom, and impsort fixes to pass CI check. --- .../datawave/accumulo/inmemory/InMemoryAccumuloClient.java | 3 +-- .../java/datawave/accumulo/inmemory/InMemoryConnector.java | 2 +- .../main/java/datawave/accumulo/inmemory/InMemoryInstance.java | 1 - 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryAccumuloClient.java b/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryAccumuloClient.java index 75d56a29bb5..9f7090ac188 100644 --- a/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryAccumuloClient.java +++ b/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryAccumuloClient.java @@ -30,7 +30,6 @@ import org.apache.accumulo.core.client.MultiTableBatchWriter; import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.client.TableNotFoundException; - import org.apache.accumulo.core.client.admin.InstanceOperations; import org.apache.accumulo.core.client.admin.NamespaceOperations; import org.apache.accumulo.core.client.admin.ReplicationOperations; @@ -69,7 +68,7 @@ public ConditionalWriter createConditionalWriter(String tableName, ConditionalWr } public ConditionalWriter createConditionalWriter(String tableName) throws TableNotFoundException { - return this.createConditionalWriter(tableName, (ConditionalWriterConfig)null); + return this.createConditionalWriter(tableName, (ConditionalWriterConfig) null); } private void ensureOpen() { diff --git a/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryConnector.java b/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryConnector.java index 37164b070a6..347a443c768 100644 --- a/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryConnector.java +++ b/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryConnector.java @@ -41,7 +41,7 @@ public class InMemoryConnector { String username; private final InMemoryAccumulo acu; - InMemoryConnector(String username, InMemoryInstance instance) throws AccumuloSecurityException { + InMemoryConnector(String username, InMemoryInstance instance) throws AccumuloSecurityException { this.username = username; this.acu = instance.acu; diff --git a/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryInstance.java b/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryInstance.java index 9e0fd620ee7..8d5e1c05c1b 100644 --- a/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryInstance.java +++ b/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryInstance.java @@ -19,7 +19,6 @@ import java.io.IOException; import java.util.Collections; import java.util.HashMap; - import java.util.List; import java.util.Map; From 0ebe50b7c92c9eadc1d529ae1b2c581087791867 Mon Sep 17 00:00:00 2001 From: Seth Date: Tue, 20 Jan 2026 11:58:42 -0500 Subject: [PATCH 03/16] Remove unused getClientContext utility from AccumuloConnectionFactory The getClientContext method used non-public ClientContext API and was only called from PushdownScheduler. Since PushdownScheduler now uses the public tableOperations().tableIdMap() API instead, this utility method is no longer needed. Supersedes PR #3340. --- .../connection/AccumuloConnectionFactory.java | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/core/connection-pool/src/main/java/datawave/core/common/connection/AccumuloConnectionFactory.java b/core/connection-pool/src/main/java/datawave/core/common/connection/AccumuloConnectionFactory.java index eb6f9ef2f99..093950374cd 100644 --- a/core/connection-pool/src/main/java/datawave/core/common/connection/AccumuloConnectionFactory.java +++ b/core/connection-pool/src/main/java/datawave/core/common/connection/AccumuloConnectionFactory.java @@ -5,10 +5,8 @@ import java.util.Map; import org.apache.accumulo.core.client.AccumuloClient; -import org.apache.accumulo.core.clientImpl.ClientContext; import datawave.core.common.result.ConnectionPool; -import datawave.webservice.common.connection.WrappedAccumuloClient; public interface AccumuloConnectionFactory extends AutoCloseable { @@ -110,21 +108,4 @@ AccumuloClient getClient(String userDN, Collection proxyServers, String * @return A map representation */ Map getTrackingMap(StackTraceElement[] stackTrace); - - /** - * Utility method to unwrap the ClientContext instance within {@link WrappedAccumuloClient} as needed - * - * @param accumuloClient - * {@link AccumuloClient} instance - * @return {@link WrappedAccumuloClient#getReal()}, if applicable; accumuloClient itself, if it implements {@link ClientContext}; otherwise returns null - */ - static ClientContext getClientContext(AccumuloClient accumuloClient) { - ClientContext cc = null; - if (accumuloClient instanceof WrappedAccumuloClient) { - cc = (ClientContext) ((WrappedAccumuloClient) accumuloClient).getReal(); - } else if (accumuloClient instanceof ClientContext) { - cc = (ClientContext) accumuloClient; - } - return cc; - } } From 03036a7afa048b62709efa34ddfb6324c7dbdefd Mon Sep 17 00:00:00 2001 From: Seth Date: Thu, 22 Jan 2026 09:02:33 -0500 Subject: [PATCH 04/16] Clarify intentionally unsupported methods in InMemoryAccumulo --- .../datawave/accumulo/inmemory/InMemoryAccumuloClient.java | 4 ++-- .../java/datawave/accumulo/inmemory/InMemoryConnector.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryAccumuloClient.java b/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryAccumuloClient.java index 9f7090ac188..6b917e40487 100644 --- a/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryAccumuloClient.java +++ b/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryAccumuloClient.java @@ -63,7 +63,7 @@ public InMemoryAccumuloClient(String username, InMemoryInstance instance) throws @Override public ConditionalWriter createConditionalWriter(String tableName, ConditionalWriterConfig config) throws TableNotFoundException { - // TODO add implementation + // Intentionally unsupported - required for AccumuloClient interface but not used by DataWave throw new UnsupportedOperationException(); } @@ -169,7 +169,7 @@ public NamespaceOperations namespaceOperations() { @Override public ReplicationOperations replicationOperations() { - // TODO add implementation + // Intentionally unsupported - required for AccumuloClient interface but not used by DataWave throw new UnsupportedOperationException(); } diff --git a/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryConnector.java b/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryConnector.java index 347a443c768..8c96711ff70 100644 --- a/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryConnector.java +++ b/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryConnector.java @@ -117,7 +117,7 @@ public NamespaceOperations namespaceOperations() { } public ConditionalWriter createConditionalWriter(String tableName, ConditionalWriterConfig config) throws TableNotFoundException { - // TODO add implementation + // Intentionally unsupported - required for Connector interface but not used by DataWave throw new UnsupportedOperationException(); } From 6737af1bdd0fb30a00ffa90b5eff6cc5f83d40b4 Mon Sep 17 00:00:00 2001 From: Ivan Bella <347158+ivakegg@users.noreply.github.com> Date: Mon, 2 Feb 2026 15:21:09 +0000 Subject: [PATCH 05/16] Add imports back in --- .../src/main/java/datawave/mr/bulk/BulkInputFormat.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/warehouse/core/src/main/java/datawave/mr/bulk/BulkInputFormat.java b/warehouse/core/src/main/java/datawave/mr/bulk/BulkInputFormat.java index 2a7c716c8d2..2700480d1dd 100644 --- a/warehouse/core/src/main/java/datawave/mr/bulk/BulkInputFormat.java +++ b/warehouse/core/src/main/java/datawave/mr/bulk/BulkInputFormat.java @@ -37,6 +37,9 @@ import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.TableOfflineException; import org.apache.accumulo.core.client.security.tokens.PasswordToken; +import org.apache.accumulo.core.clientImpl.ClientConfConverter; +import org.apache.accumulo.core.clientImpl.ClientContext; +import org.apache.accumulo.core.clientImpl.ClientInfo; import org.apache.accumulo.core.clientImpl.TabletLocator; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.PartialKey; @@ -47,12 +50,15 @@ import org.apache.accumulo.core.iterators.SortedKeyValueIterator; import org.apache.accumulo.core.iterators.user.RegExFilter; import org.apache.accumulo.core.iterators.user.VersioningIterator; +import org.apache.accumulo.core.manager.state.tables.TableState; import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.schema.MetadataSchema; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.accumulo.core.security.TablePermission; +import org.apache.accumulo.core.singletons.SingletonManager; import org.apache.accumulo.core.util.format.DateFormatSupplier; +import org.apache.accumulo.core.util.threads.Threads; import org.apache.commons.codec.binary.Base64; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.conf.Configuration; From d66e1f0aa89a4ca548d1e8cd1bf16ba05adb0f57 Mon Sep 17 00:00:00 2001 From: Ivan Bella <347158+ivakegg@users.noreply.github.com> Date: Mon, 2 Feb 2026 15:41:43 +0000 Subject: [PATCH 06/16] Added in too many --- .../core/src/main/java/datawave/mr/bulk/BulkInputFormat.java | 1 - 1 file changed, 1 deletion(-) diff --git a/warehouse/core/src/main/java/datawave/mr/bulk/BulkInputFormat.java b/warehouse/core/src/main/java/datawave/mr/bulk/BulkInputFormat.java index 2700480d1dd..51b31627b1a 100644 --- a/warehouse/core/src/main/java/datawave/mr/bulk/BulkInputFormat.java +++ b/warehouse/core/src/main/java/datawave/mr/bulk/BulkInputFormat.java @@ -50,7 +50,6 @@ import org.apache.accumulo.core.iterators.SortedKeyValueIterator; import org.apache.accumulo.core.iterators.user.RegExFilter; import org.apache.accumulo.core.iterators.user.VersioningIterator; -import org.apache.accumulo.core.manager.state.tables.TableState; import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.schema.MetadataSchema; import org.apache.accumulo.core.security.Authorizations; From d7d4e811ed41c38ab7f8d4a3fd7d67211d3abb98 Mon Sep 17 00:00:00 2001 From: Seth Date: Tue, 28 Apr 2026 11:03:19 -0400 Subject: [PATCH 07/16] Address lbschanno and ddanielr review feedback InMemoryAccumuloClient: - Remove unused ConditionalWriterConfig field and ensureOpen method - Remove AccumuloSecurityException from constructor (not thrown) - Remove unused createConditionalWriter(String) overload - Remove TableNotFoundException from createConditionalWriter (not thrown) - Move inline comments to javadoc InMemoryConnector: - Remove AccumuloSecurityException from constructor (not thrown) - Move inline comment to javadoc BulkInputFormat: - Simplify online check per ddanielr: replace tableId/tableIdMap/isOnline block with single isOnline(tableName) call - Remove unused tableId variable and TableDeletedException import Part of #2443 --- .../inmemory/InMemoryAccumuloClient.java | 30 +++++++------------ .../accumulo/inmemory/InMemoryConnector.java | 7 +++-- .../datawave/mr/bulk/BulkInputFormat.java | 11 ++----- 3 files changed, 17 insertions(+), 31 deletions(-) diff --git a/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryAccumuloClient.java b/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryAccumuloClient.java index 6b917e40487..d7d40b9f523 100644 --- a/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryAccumuloClient.java +++ b/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryAccumuloClient.java @@ -39,19 +39,15 @@ import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.SystemPermission; -// Remove `extends ClientContext` since it's deprecated. -// Need a workaround for the features that are no longer supported. Not sure if we can just throw them away atm. - +/** + * In-memory implementation of {@link AccumuloClient} for testing. Does not connect to a real Accumulo instance. + */ public class InMemoryAccumuloClient implements AccumuloClient { String username; private final InMemoryAccumulo acu; - private ConditionalWriterConfig conditionalWriterConfig; - private volatile boolean closed = false; - - public InMemoryAccumuloClient(String username, InMemoryInstance instance) throws AccumuloSecurityException { - + public InMemoryAccumuloClient(String username, InMemoryInstance instance) { this.username = username; this.acu = instance.acu; if (!acu.users.containsKey(username)) { @@ -61,22 +57,14 @@ public InMemoryAccumuloClient(String username, InMemoryInstance instance) throws } } + /** + * Not supported by the in-memory implementation. + */ @Override public ConditionalWriter createConditionalWriter(String tableName, ConditionalWriterConfig config) throws TableNotFoundException { - // Intentionally unsupported - required for AccumuloClient interface but not used by DataWave throw new UnsupportedOperationException(); } - public ConditionalWriter createConditionalWriter(String tableName) throws TableNotFoundException { - return this.createConditionalWriter(tableName, (ConditionalWriterConfig) null); - } - - private void ensureOpen() { - if (this.closed) { - throw new IllegalStateException("This client was closed."); - } - } - @Override public BatchScanner createBatchScanner(String tableName, Authorizations authorizations, int numQueryThreads) throws TableNotFoundException { if (acu.tables.get(tableName) == null) @@ -167,9 +155,11 @@ public NamespaceOperations namespaceOperations() { return new InMemoryNamespaceOperations(acu, username); } + /** + * Not supported by the in-memory implementation. + */ @Override public ReplicationOperations replicationOperations() { - // Intentionally unsupported - required for AccumuloClient interface but not used by DataWave throw new UnsupportedOperationException(); } diff --git a/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryConnector.java b/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryConnector.java index 8c96711ff70..a25d3e04f5e 100644 --- a/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryConnector.java +++ b/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryConnector.java @@ -41,8 +41,7 @@ public class InMemoryConnector { String username; private final InMemoryAccumulo acu; - InMemoryConnector(String username, InMemoryInstance instance) throws AccumuloSecurityException { - + InMemoryConnector(String username, InMemoryInstance instance) { this.username = username; this.acu = instance.acu; if (!acu.users.containsKey(username)) { @@ -116,8 +115,10 @@ public NamespaceOperations namespaceOperations() { return new InMemoryNamespaceOperations(acu, username); } + /** + * Not supported by the in-memory implementation. + */ public ConditionalWriter createConditionalWriter(String tableName, ConditionalWriterConfig config) throws TableNotFoundException { - // Intentionally unsupported - required for Connector interface but not used by DataWave throw new UnsupportedOperationException(); } diff --git a/warehouse/core/src/main/java/datawave/mr/bulk/BulkInputFormat.java b/warehouse/core/src/main/java/datawave/mr/bulk/BulkInputFormat.java index 4750bd0fe4e..e21196bca24 100644 --- a/warehouse/core/src/main/java/datawave/mr/bulk/BulkInputFormat.java +++ b/warehouse/core/src/main/java/datawave/mr/bulk/BulkInputFormat.java @@ -31,7 +31,6 @@ import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.client.RowIterator; import org.apache.accumulo.core.client.Scanner; -import org.apache.accumulo.core.client.TableDeletedException; import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.TableOfflineException; import org.apache.accumulo.core.client.security.tokens.PasswordToken; @@ -1094,7 +1093,6 @@ public List getSplits(JobContext job) throws IOException { } } else { try (AccumuloClient client = getClient(job.getConfiguration())) { - TableId tableId = null; tl = getTabletLocator(job.getConfiguration()); // its possible that the cache could contain complete, but old information about a tables tablets... so clear it tl.invalidateCache(); @@ -1103,12 +1101,9 @@ public List getSplits(JobContext job) throws IOException { ClientConfConverter.toAccumuloConf(info.getProperties()), Threads.UEH); while (!tl.binRanges(context, ranges, binnedRanges).isEmpty()) { if (!(client instanceof InMemoryAccumuloClient)) { - if (tableId == null) - tableId = TableId.of(client.tableOperations().tableIdMap().get(tableName));// TABLE ID todo:ensure this is correct - if (!client.tableOperations().tableIdMap().containsKey(tableName))// CHECK IF TABLE ID EXISTS //todo same - throw new TableDeletedException(tableId.canonical()); - if (!client.tableOperations().isOnline(tableName)) // CHECK IF TABLE ID OFFLInE //todo same - throw new TableOfflineException("Table (" + tableId.canonical() + ") is offline"); + if (!client.tableOperations().isOnline(tableName)) { + throw new TableOfflineException((TableId) null, tableName); + } } binnedRanges.clear(); log.warn("Unable to locate bins for specified ranges. Retrying."); From e7f210d5e1ff239a8c982ff659b96e3694dde59b Mon Sep 17 00:00:00 2001 From: Seth Date: Tue, 5 May 2026 17:12:19 -0400 Subject: [PATCH 08/16] Remove unused AccumuloSecurityException import from InMemoryConnector --- .../main/java/datawave/accumulo/inmemory/InMemoryConnector.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryConnector.java b/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryConnector.java index a25d3e04f5e..ca8ca18b8c3 100644 --- a/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryConnector.java +++ b/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryConnector.java @@ -18,7 +18,6 @@ import java.util.concurrent.TimeUnit; -import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.BatchDeleter; import org.apache.accumulo.core.client.BatchScanner; import org.apache.accumulo.core.client.BatchWriter; From 834df621f9caa778b7ae3ed7359e115c02fd08c8 Mon Sep 17 00:00:00 2001 From: Seth Date: Wed, 6 May 2026 12:21:10 -0400 Subject: [PATCH 09/16] Restore createConditionalWriter(String) required by AccumuloClient interface --- .../accumulo/inmemory/InMemoryAccumuloClient.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryAccumuloClient.java b/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryAccumuloClient.java index d7d40b9f523..608887560b5 100644 --- a/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryAccumuloClient.java +++ b/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryAccumuloClient.java @@ -57,6 +57,14 @@ public InMemoryAccumuloClient(String username, InMemoryInstance instance) { } } + /** + * Not supported by the in-memory implementation. + */ + @Override + public ConditionalWriter createConditionalWriter(String tableName) throws TableNotFoundException { + throw new UnsupportedOperationException(); + } + /** * Not supported by the in-memory implementation. */ From d5f90a045c5bdb1fb49053bed00e1ba621e7ea92 Mon Sep 17 00:00:00 2001 From: Seth Date: Tue, 12 May 2026 10:26:46 -0400 Subject: [PATCH 10/16] Make username field final, remove unnecessary throws declarations --- .../datawave/accumulo/inmemory/InMemoryAccumuloClient.java | 6 +++--- .../java/datawave/accumulo/inmemory/InMemoryConnector.java | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryAccumuloClient.java b/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryAccumuloClient.java index 608887560b5..76e6b62f948 100644 --- a/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryAccumuloClient.java +++ b/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryAccumuloClient.java @@ -44,7 +44,7 @@ */ public class InMemoryAccumuloClient implements AccumuloClient { - String username; + final String username; private final InMemoryAccumulo acu; public InMemoryAccumuloClient(String username, InMemoryInstance instance) { @@ -61,7 +61,7 @@ public InMemoryAccumuloClient(String username, InMemoryInstance instance) { * Not supported by the in-memory implementation. */ @Override - public ConditionalWriter createConditionalWriter(String tableName) throws TableNotFoundException { + public ConditionalWriter createConditionalWriter(String tableName) { throw new UnsupportedOperationException(); } @@ -69,7 +69,7 @@ public ConditionalWriter createConditionalWriter(String tableName) throws TableN * Not supported by the in-memory implementation. */ @Override - public ConditionalWriter createConditionalWriter(String tableName, ConditionalWriterConfig config) throws TableNotFoundException { + public ConditionalWriter createConditionalWriter(String tableName, ConditionalWriterConfig config) { throw new UnsupportedOperationException(); } diff --git a/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryConnector.java b/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryConnector.java index ca8ca18b8c3..7ee4abb4eb6 100644 --- a/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryConnector.java +++ b/core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryConnector.java @@ -37,7 +37,7 @@ public class InMemoryConnector { - String username; + final String username; private final InMemoryAccumulo acu; InMemoryConnector(String username, InMemoryInstance instance) { From 2295bd78b5bb879c9bf61d73b74eb8429edcd533 Mon Sep 17 00:00:00 2001 From: Seth Date: Tue, 12 May 2026 11:19:42 -0400 Subject: [PATCH 11/16] Bump in-memory-accumulo to 4.0.6-SNAPSHOT and fix MockMetadataHelper Constructor no longer declares AccumuloSecurityException, so remove the dead try/catch in MockMetadataHelper.getClient(). --- pom.xml | 2 +- .../test/java/datawave/query/util/MockMetadataHelper.java | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index 44d20522570..64e5be16ae9 100644 --- a/pom.xml +++ b/pom.xml @@ -99,7 +99,7 @@ 1.3 4.5.13 4.4.8 - 4.0.4 + 4.0.6-SNAPSHOT 9.4.21.Final 2.13.5 1.9.13 diff --git a/warehouse/query-core/src/test/java/datawave/query/util/MockMetadataHelper.java b/warehouse/query-core/src/test/java/datawave/query/util/MockMetadataHelper.java index ab09211e098..76fd56e1532 100644 --- a/warehouse/query-core/src/test/java/datawave/query/util/MockMetadataHelper.java +++ b/warehouse/query-core/src/test/java/datawave/query/util/MockMetadataHelper.java @@ -84,11 +84,7 @@ private static AllFieldMetadataHelper createAllFieldMetadataHelper(AccumuloClien } private static AccumuloClient getClient() { - try { - return new InMemoryAccumuloClient("root", new InMemoryInstance()); - } catch (AccumuloSecurityException e) { - throw new RuntimeException(e); - } + return new InMemoryAccumuloClient("root", new InMemoryInstance()); } public void addContentFields(Collection fields) { From 54cb9d9500b924343eee8c6945a030b0a9d800e9 Mon Sep 17 00:00:00 2001 From: Seth Date: Wed, 20 May 2026 16:42:08 -0400 Subject: [PATCH 12/16] Add /etc/hadoop/conf symlink in quickstart Docker image MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Datawave's server-side query iterators load Hadoop configuration from /etc/hadoop/conf when they run inside the Accumulo TabletServer. That path does not exist in the quickstart container — Hadoop is installed under /opt/datawave/contrib/datawave-quickstart/hadoop and HADOOP_CONF_DIR points there, not at /etc/hadoop/conf. Without the path, every event-table scan throws FileNotFoundException server-side and the query service silently returns 0 events. Symlink /etc/hadoop/conf to the installed Hadoop config directory so the expected path resolves. Verified locally: docker/scripts/testAll.sh goes from 5/13 to 13/13 passing. --- contrib/datawave-quickstart/docker/Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/contrib/datawave-quickstart/docker/Dockerfile b/contrib/datawave-quickstart/docker/Dockerfile index 933362b3bb0..acc538a4301 100644 --- a/contrib/datawave-quickstart/docker/Dockerfile +++ b/contrib/datawave-quickstart/docker/Dockerfile @@ -120,6 +120,11 @@ RUN /bin/bash -c "/usr/bin/nohup /usr/sbin/sshd -D &" && \ rm -rf contrib/datawave-quickstart/accumulo/logs/* && \ rm -rf contrib/datawave-quickstart/wildfly/standalone/log/* +# Accumulo TabletServer-side datawave iterators load Hadoop config from /etc/hadoop/conf; +# without this symlink, event-table scans error with FileNotFoundException server-side. +RUN mkdir -p /etc/hadoop && \ + ln -sfn /opt/datawave/contrib/datawave-quickstart/hadoop/etc/hadoop /etc/hadoop/conf + # Lastly, establish volumes for data, logs & other directories, wire up # the entrypoint & bootstrap scripts, expose ports, and set default CMD... From 785f751ae84ef607c8648d1ca8afd7ace4cb11c6 Mon Sep 17 00:00:00 2001 From: Seth Date: Wed, 20 May 2026 16:42:08 -0400 Subject: [PATCH 13/16] Add quickstart dependency for services that connect to ZooKeeper or Hadoop Nine services in docker-compose.yml use ZOOKEEPER_HOST or HADOOP_HOST but did not declare a depends_on for the quickstart container, which hosts ZooKeeper, HDFS, and Accumulo. Only metrics correctly declared the dependency. When quickstart's startup happens to take longer than the 60s ZooKeeper connect timeout used by Spring services like audit's AccumuloAuditor bean, those services exit 1 during init with "Failed to connect to zookeeper (quickstart:2181) within 2x zookeeper timeout period 30000", and docker compose aborts with "dependency failed to start: container docker-audit-1 exited (1)". The race is environment-dependent: on slower runners it is frequently lost, producing intermittent CI failures. Declare depends_on quickstart (condition: service_healthy) for annotation, accumulo, audit, dictionary, modification, query, mapreduce-query, executor-pool1, and executor-pool2 so the startup is correctly serialized behind the container that provides their backing services. --- docker/docker-compose.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 7a014326b67..4c1943efe18 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -221,6 +221,8 @@ services: depends_on: configuration: condition: service_healthy + quickstart: + condition: service_healthy query-cache: profiles: @@ -361,6 +363,8 @@ services: depends_on: authorization: condition: service_healthy + quickstart: + condition: service_healthy audit: image: nationalsecurityagency/datawave/audit-service:${VERSION:-latest} @@ -394,6 +398,8 @@ services: depends_on: authorization: condition: service_healthy + quickstart: + condition: service_healthy metrics: entrypoint: ["java","-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5007","-jar","app.jar"] @@ -463,6 +469,8 @@ services: depends_on: authorization: condition: service_healthy + quickstart: + condition: service_healthy file-provider: profiles: @@ -545,6 +553,8 @@ services: condition: service_healthy executor-pool1: condition: service_started + quickstart: + condition: service_healthy mapreduce-query: profiles: @@ -583,6 +593,8 @@ services: condition: service_healthy executor-pool1: condition: service_started + quickstart: + condition: service_healthy executor-pool1: entrypoint: ["java","-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5006","-jar","app.jar"] @@ -630,6 +642,8 @@ services: condition: service_healthy metrics: condition: service_healthy + quickstart: + condition: service_healthy executor-pool2: profiles: @@ -678,6 +692,8 @@ services: condition: service_healthy metrics: condition: service_healthy + quickstart: + condition: service_healthy modification: entrypoint: ["java","-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5010","-jar","app.jar"] @@ -712,6 +728,8 @@ services: depends_on: authorization: condition: service_healthy + quickstart: + condition: service_healthy # If you use the management center, you can connect to the hazelcast cache as follows: # In your browser connect to https://localhost:9243/ From 64e8cbcf97157c644e56a70c6577b7b6c6ba2922 Mon Sep 17 00:00:00 2001 From: Seth Date: Thu, 21 May 2026 14:55:25 -0400 Subject: [PATCH 14/16] Bump spring-boot-starter-datawave-query consumers to 1.0.11-SNAPSHOT PR #3560 removed deprecated eventPerDayThreshold and shardsPerDayThreshold placeholders from the starter source and from docker/config/application-query.yml, but the released spring-boot-starter-datawave-query-1.0.10.jar still embeds the old QueryLogicFactory.xml that references those placeholders. With no fresh 1.0.11 release cut yet, every consumer of the 1.0.10 jar (query, executor, mapreduce-query, cached-results) fails Spring init at startup with: Caused by: java.lang.IllegalArgumentException: Could not resolve placeholder 'datawave.query.logic.logics.BaseEventQuery.eventPerDayThreshold' This blocks the compose-tests workflow on every recent PR opened against integration (#3559, #3562, #3563, #3565 all show the same FAILURE). Bump from 1.0.10 to 1.0.11-SNAPSHOT in the six consuming poms so the multi-module reactor builds the starter from current source (which has #3560's removal) and the query/executor docker images embed the fresh jar. This is a stop-gap. The proper fix is running microservice-cascade-release.yml starting from microservices/starters/query to publish a real 1.0.11; once that lands, these SNAPSHOT pins should be replaced with the released version. --- microservices/services/mapreduce-query/jobs/core/pom.xml | 2 +- microservices/services/mapreduce-query/service/pom.xml | 2 +- microservices/services/query-executor/api/pom.xml | 2 +- microservices/services/query-executor/service/pom.xml | 2 +- microservices/services/query/service/pom.xml | 2 +- microservices/starters/cached-results/pom.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/microservices/services/mapreduce-query/jobs/core/pom.xml b/microservices/services/mapreduce-query/jobs/core/pom.xml index c4c38d98024..1032004f0a5 100644 --- a/microservices/services/mapreduce-query/jobs/core/pom.xml +++ b/microservices/services/mapreduce-query/jobs/core/pom.xml @@ -20,7 +20,7 @@ 7.33.1 1.0.0 - 1.0.10 + 1.0.11-SNAPSHOT 3.9.2 diff --git a/microservices/services/mapreduce-query/service/pom.xml b/microservices/services/mapreduce-query/service/pom.xml index 672dbcb0ca4..14e11b7bba5 100644 --- a/microservices/services/mapreduce-query/service/pom.xml +++ b/microservices/services/mapreduce-query/service/pom.xml @@ -26,7 +26,7 @@ 4.0.2 1.0.5-SNAPSHOT 4.0.7 - 1.0.10 + 1.0.11-SNAPSHOT 3.3.4 2.25.3 diff --git a/microservices/services/query-executor/api/pom.xml b/microservices/services/query-executor/api/pom.xml index 84313acbedf..872e5c61772 100644 --- a/microservices/services/query-executor/api/pom.xml +++ b/microservices/services/query-executor/api/pom.xml @@ -20,7 +20,7 @@ http://webservice.datawave/v1 3.20.0 7.33.1 - 1.0.10 + 1.0.11-SNAPSHOT 2.3.3 diff --git a/microservices/services/query-executor/service/pom.xml b/microservices/services/query-executor/service/pom.xml index bc7cd01bef2..322a4fb91ad 100644 --- a/microservices/services/query-executor/service/pom.xml +++ b/microservices/services/query-executor/service/pom.xml @@ -29,7 +29,7 @@ 1.0.9 4.1.3 4.0.7 - 1.0.10 + 1.0.11-SNAPSHOT 3.0.5 31.1-jre 3.3.4 diff --git a/microservices/services/query/service/pom.xml b/microservices/services/query/service/pom.xml index 4ee665c5a92..0e17b0b9db7 100644 --- a/microservices/services/query/service/pom.xml +++ b/microservices/services/query/service/pom.xml @@ -33,7 +33,7 @@ 4.0.7 4.0.5 1.0.12 - 1.0.10 + 1.0.11-SNAPSHOT 3.0.5 31.1-jre 3.3.4 diff --git a/microservices/starters/cached-results/pom.xml b/microservices/starters/cached-results/pom.xml index d160e2bff1c..75b5648f31c 100644 --- a/microservices/starters/cached-results/pom.xml +++ b/microservices/starters/cached-results/pom.xml @@ -33,7 +33,7 @@ 1.0.1 4.0.7 4.0.5 - 1.0.10 + 1.0.11-SNAPSHOT 9.3.0 From 9ca3cad49e95db513daa7f8a820499fc095f9940 Mon Sep 17 00:00:00 2001 From: foster33 <84727868+foster33@users.noreply.github.com> Date: Tue, 2 Jun 2026 18:34:05 +0000 Subject: [PATCH 15/16] Catch AccumuloSecurityException in MockMetadataHelper --- .../test/java/datawave/query/util/MockMetadataHelper.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/warehouse/query-core/src/test/java/datawave/query/util/MockMetadataHelper.java b/warehouse/query-core/src/test/java/datawave/query/util/MockMetadataHelper.java index 76fd56e1532..ab09211e098 100644 --- a/warehouse/query-core/src/test/java/datawave/query/util/MockMetadataHelper.java +++ b/warehouse/query-core/src/test/java/datawave/query/util/MockMetadataHelper.java @@ -84,7 +84,11 @@ private static AllFieldMetadataHelper createAllFieldMetadataHelper(AccumuloClien } private static AccumuloClient getClient() { - return new InMemoryAccumuloClient("root", new InMemoryInstance()); + try { + return new InMemoryAccumuloClient("root", new InMemoryInstance()); + } catch (AccumuloSecurityException e) { + throw new RuntimeException(e); + } } public void addContentFields(Collection fields) { From a691897be6db474f2b7fd9728d85606591db7f87 Mon Sep 17 00:00:00 2001 From: foster33 <84727868+foster33@users.noreply.github.com> Date: Tue, 2 Jun 2026 19:12:44 +0000 Subject: [PATCH 16/16] Remove latest exception changes --- .../test/java/datawave/query/util/MockMetadataHelper.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/warehouse/query-core/src/test/java/datawave/query/util/MockMetadataHelper.java b/warehouse/query-core/src/test/java/datawave/query/util/MockMetadataHelper.java index ab09211e098..76fd56e1532 100644 --- a/warehouse/query-core/src/test/java/datawave/query/util/MockMetadataHelper.java +++ b/warehouse/query-core/src/test/java/datawave/query/util/MockMetadataHelper.java @@ -84,11 +84,7 @@ private static AllFieldMetadataHelper createAllFieldMetadataHelper(AccumuloClien } private static AccumuloClient getClient() { - try { - return new InMemoryAccumuloClient("root", new InMemoryInstance()); - } catch (AccumuloSecurityException e) { - throw new RuntimeException(e); - } + return new InMemoryAccumuloClient("root", new InMemoryInstance()); } public void addContentFields(Collection fields) {