Skip to content
Open
Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
1c86b79
Replace ClientContext.getTableId with public API
SethSmucker Jan 5, 2026
5f29ebf
Add AccumuloTableInfoFetcher facade to replace non-public Accumulo AP…
SethSmucker Mar 2, 2026
0172f51
Address PR #3449 review feedback from jschmidt10
SethSmucker Mar 23, 2026
2c33a78
Merge
SethSmucker Mar 23, 2026
a488b60
Merge branch 'integration' into task/clientcontext-facade-migration
SethSmucker Mar 24, 2026
0c04d33
Use Thrift RPC for queued compaction counts, inline public API calls …
SethSmucker Mar 27, 2026
44e5aa0
Merge branch 'integration' into task/clientcontext-facade-migration
SethSmucker Mar 27, 2026
be1a2e1
Merge remote-tracking branch 'origin/integration' into task/clientcon…
SethSmucker Mar 27, 2026
31bb688
Merge branch 'task/clientcontext-facade-migration' of github.com:Nati…
SethSmucker Mar 27, 2026
4ffad05
Merge branch 'integration' into task/clientcontext-facade-migration
SethSmucker Mar 30, 2026
619d710
Refactor AccumuloTableInfoFetcher to static utility class
SethSmucker Apr 6, 2026
0e166e9
Merge branch 'integration' into task/clientcontext-facade-migration
SethSmucker Apr 10, 2026
516f3c6
Fix javadoc generation for AccumuloTableInfoFetcher.locateTablets
SethSmucker Apr 10, 2026
d2c0891
Merge branch 'integration' into task/clientcontext-facade-migration
SethSmucker Apr 14, 2026
40f003f
Merge remote-tracking branch 'origin/integration' into task/clientcon…
SethSmucker Apr 22, 2026
19772a9
Merge branch 'integration' into task/clientcontext-facade-migration
lbschanno Apr 24, 2026
60f25db
Rename AccumuloTableInfoFetcher to AccumuloTableUtils per lbschanno
SethSmucker Apr 28, 2026
87c1cfe
Remove AccumuloSecurityException from throws clauses per lbschanno re…
SethSmucker May 5, 2026
866f286
Add javadoc to clipRanges() per lbschanno review
SethSmucker May 5, 2026
3089e5e
Merge branch 'integration' into task/clientcontext-facade-migration
SethSmucker May 12, 2026
1c463ca
Merge branch 'integration' into task/clientcontext-facade-migration
SethSmucker May 18, 2026
7428623
Merge branch 'integration' into task/clientcontext-facade-migration
SethSmucker May 19, 2026
b243082
Merge branch 'integration' into task/clientcontext-facade-migration
SethSmucker May 20, 2026
54cb9d9
Add /etc/hadoop/conf symlink in quickstart Docker image
SethSmucker May 20, 2026
785f751
Add quickstart dependency for services that connect to ZooKeeper or H…
SethSmucker May 20, 2026
e20b985
Merge branch 'integration' into task/compose-tests-stability
SethSmucker May 20, 2026
64e8cbc
Bump spring-boot-starter-datawave-query consumers to 1.0.11-SNAPSHOT
SethSmucker May 21, 2026
50b27a9
Merge task/compose-tests-stability into task/clientcontext-facade-mig…
SethSmucker Jun 1, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,8 @@
import java.util.Map;

import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.clientImpl.ClientContext;

import datawave.core.common.result.ConnectionPool;
import datawave.webservice.common.connection.WrappedAccumuloClient;

public interface AccumuloConnectionFactory extends AutoCloseable {

Expand Down Expand Up @@ -110,21 +108,4 @@ AccumuloClient getClient(String userDN, Collection<String> proxyServers, String
* @return A map representation
*/
Map<String,String> getTrackingMap(StackTraceElement[] stackTrace);

/**
* Utility method to unwrap the ClientContext instance within {@link WrappedAccumuloClient} as needed
*
* @param accumuloClient
* {@link AccumuloClient} instance
* @return {@link WrappedAccumuloClient#getReal()}, if applicable; accumuloClient itself, if it implements {@link ClientContext}; otherwise returns null
*/
static ClientContext getClientContext(AccumuloClient accumuloClient) {
ClientContext cc = null;
if (accumuloClient instanceof WrappedAccumuloClient) {
cc = (ClientContext) ((WrappedAccumuloClient) accumuloClient).getReal();
} else if (accumuloClient instanceof ClientContext) {
cc = (ClientContext) accumuloClient;
}
return cc;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
package datawave.core.common.connection;

import java.util.Collection;
import java.util.List;

import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.admin.ActiveCompaction;
import org.apache.accumulo.core.client.admin.Locations;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.TableId;

/**
* Facade that centralizes Accumulo table metadata operations behind public APIs.
* <p>
* This class replaces direct usage of non-public Accumulo internals ({@code ClientContext}, {@code ThriftClientTypes}, {@code TabletLocator},
* {@code MetadataServicer}, etc.) with their public API equivalents. All methods delegate to {@link org.apache.accumulo.core.client.admin.TableOperations} or
* {@link org.apache.accumulo.core.client.admin.InstanceOperations}.
*
* @see <a href="https://github.com/NationalSecurityAgency/datawave/issues/2443">Issue #2443</a>
*/
public class AccumuloTableInfoFetcher {
Comment thread
SethSmucker marked this conversation as resolved.
Outdated

private final AccumuloClient client;

public AccumuloTableInfoFetcher(AccumuloClient client) {
this.client = client;
}

/**
* Get the TableId for a table name using the public {@code tableIdMap()} API.
Comment thread
SethSmucker marked this conversation as resolved.
Outdated
*
* @param tableName
* the table name
* @return the TableId
* @throws TableNotFoundException
* if the table does not exist
*/
public TableId getTableId(String tableName) throws TableNotFoundException {
String id = client.tableOperations().tableIdMap().get(tableName);
if (id == null) {
throw new TableNotFoundException(null, tableName, "Table not found in tableIdMap");
}
return TableId.of(id);
}

/**
* Check if a table exists using the public {@code exists()} API.
*
* @param tableName
* the table name
* @return true if the table exists
*/
public boolean tableExists(String tableName) {
return client.tableOperations().exists(tableName);
}

/**
* Check if a table is online using the public {@code isOnline()} API.
*
* @param tableName
* the table name
* @return true if the table is online
* @throws TableNotFoundException
* if the table does not exist
* @throws AccumuloException
* if a general Accumulo error occurs
*/
public boolean isTableOnline(String tableName) throws TableNotFoundException, AccumuloException {
return client.tableOperations().isOnline(tableName);
}

/**
* Get tablet locations for the given ranges using the public {@code locate()} API.
*
* @param tableName
* the table name
* @param ranges
* the ranges to locate
* @return the Locations result
* @throws TableNotFoundException
* if the table does not exist
* @throws AccumuloException
* if a general Accumulo error occurs
* @throws AccumuloSecurityException
* if a security error occurs
*/
public Locations getTabletLocations(String tableName, Collection<Range> ranges)
throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
return client.tableOperations().locate(tableName, ranges);
}

/**
* Get the count of running major compactions across all tablet servers using the public {@code getActiveCompactions()} API.
* <p>
* Note: This counts only running compactions (not queued), which differs slightly from the original Thrift-based implementation that also counted queued
* compactions. This is acceptable because the MAJC_THRESHOLD default is 3000 (a high safety margin) and this is polled on each bulk load cycle.
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure relying solely on active will work for us. On some of the instances, the tservers never really stop compacting so having a threshold on the active would either always trigger or never trigger. The queued really illustrates the backlog. Thoughts @ivakegg @hlgp ?

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@SethSmucker are there any available APIs to get queued compactions? If so, can we just add it to maintain the historical behavior?

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not that I could find for 2.1.* at least, but it looks like it will be available once we move to 4.0. For now I'll use the same facade pattern and we can swap out the internals once we hit 4.0, removing the facades one by one so we can test them on the high side. I'll push a draft for that soon, but if there's another way to do it that would work better I'm all ears

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also, it looked like MetadataServicer might not need a class replacement, the public API versions of its usecase are fairly straight forward, so I think we can directly call them instead of having the original facade. We'll still have the one I mentioned above for the compactions, but just not the original one from the ticket. If it would be better to keep it as a centralized place (closer to what MetadataServicer was doing) I can revert it, but I'll go ahead and try out the direct method calls for now.

*
* @return the number of active major compactions
* @throws AccumuloException
* if a general Accumulo error occurs
* @throws AccumuloSecurityException
* if a security error occurs
*/
public int getMajorCompactionCount() throws AccumuloException, AccumuloSecurityException {
int count = 0;
List<ActiveCompaction> compactions = client.instanceOperations().getActiveCompactions();
for (ActiveCompaction compaction : compactions) {
if (compaction.getType() == ActiveCompaction.CompactionType.MAJOR || compaction.getType() == ActiveCompaction.CompactionType.FULL) {
count++;
}
}
return count;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
package datawave.core.common.connection;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;

import java.util.Collections;
import java.util.List;
import java.util.Map;

import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.admin.Locations;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.data.TabletId;
import org.junit.Before;
import org.junit.Test;

import datawave.accumulo.inmemory.InMemoryAccumuloClient;
import datawave.accumulo.inmemory.InMemoryInstance;

public class AccumuloTableInfoFetcherTest {

private static final String TEST_TABLE = "testTable";
private AccumuloClient client;
private AccumuloTableInfoFetcher fetcher;

@Before
public void setup() throws Exception {
InMemoryInstance instance = new InMemoryInstance();
client = new InMemoryAccumuloClient("root", instance);
client.tableOperations().create(TEST_TABLE);
fetcher = new AccumuloTableInfoFetcher(client);
}

@Test
public void testGetTableId() throws Exception {
TableId tableId = fetcher.getTableId(TEST_TABLE);
assertNotNull(tableId);
// Verify it matches the tableIdMap entry
String expectedId = client.tableOperations().tableIdMap().get(TEST_TABLE);
assertEquals(expectedId, tableId.canonical());
}

@Test(expected = TableNotFoundException.class)
public void testGetTableIdNonExistent() throws Exception {
fetcher.getTableId("nonExistentTable");
}

@Test
public void testTableExists() {
assertTrue(fetcher.tableExists(TEST_TABLE));
assertFalse(fetcher.tableExists("nonExistentTable"));
}

@Test
public void testIsTableOnline() throws Exception {
// InMemoryAccumuloClient.isOnline() returns false, but the API call should succeed
boolean online = fetcher.isTableOnline(TEST_TABLE);
// InMemory always returns false; just verify no exception is thrown
assertFalse(online);
}

@Test
public void testGetTabletLocations() throws Exception {
Locations locations = fetcher.getTabletLocations(TEST_TABLE, Collections.singletonList(new Range()));
assertNotNull(locations);
Map<TabletId,List<Range>> byTablet = locations.groupByTablet();
assertNotNull(byTablet);
assertFalse(byTablet.isEmpty());
}

@Test
public void testGetMajorCompactionCount() throws Exception {
// InMemoryAccumuloClient returns empty list for getActiveCompactions
int count = fetcher.getMajorCompactionCount();
assertEquals(0, count);
}
}
3 changes: 1 addition & 2 deletions import-control-accumulo.xml
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,8 @@
<allow pkg="org.apache.accumulo.core.spi"/>

<!-- Temporarily allow some specific classes until accumulo
team seperates parts of that into public api -->
team separates parts of that into public api -->
<allow class="org.apache.accumulo.core.conf.Property"/>
<allow class="org.apache.accumulo.core.clientImpl.TabletLocator"/>
<allow class="org.apache.accumulo.core.file.keyfunctor.KeyFunctor" />
<!-- disallow everything else coming from accumulo -->
<disallow pkg="org.apache.accumulo"/>
Expand Down
5 changes: 5 additions & 0 deletions warehouse/core/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,11 @@
<groupId>gov.nsa.datawave.core</groupId>
<artifactId>common-utils</artifactId>
</dependency>
<dependency>
<groupId>gov.nsa.datawave.core</groupId>
<artifactId>datawave-core-connection-pool</artifactId>
<version>${project.version}</version>
Comment thread
SethSmucker marked this conversation as resolved.
Outdated
</dependency>
<dependency>
<groupId>gov.nsa.datawave.core</groupId>
<artifactId>metadata-utils</artifactId>
Expand Down
Loading
Loading