diff --git a/core/pom.xml b/core/pom.xml index 0f44bf219d6..d9b9c244a6e 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -21,6 +21,7 @@ query utils base-rest-responses + tables diff --git a/core/tables/pom.xml b/core/tables/pom.xml new file mode 100644 index 00000000000..fa6d896ea34 --- /dev/null +++ b/core/tables/pom.xml @@ -0,0 +1,24 @@ + + + 4.0.0 + + gov.nsa.datawave.core + datawave-core-parent + 7.40.0-SNAPSHOT + + + datawave-core-tables + + + 11 + 11 + UTF-8 + + + + + org.apache.hadoop + hadoop-client-api + + + diff --git a/core/tables/src/main/java/datawave/table/constants/ColumnFamilyConstants.java b/core/tables/src/main/java/datawave/table/constants/ColumnFamilyConstants.java new file mode 100644 index 00000000000..60e804d2c0c --- /dev/null +++ b/core/tables/src/main/java/datawave/table/constants/ColumnFamilyConstants.java @@ -0,0 +1,19 @@ +package datawave.table.constants; + +import org.apache.hadoop.io.Text; + +/** + * Constants for Accumulo ColumnFamilies reserved by DataWave + */ +public class ColumnFamilyConstants { + + public static final String TERM_FREQUENCY = "tf"; + public static final String FULL_CONTENT = "d"; + + public static final Text TERM_FREQUENCY_TEXT = new Text(TERM_FREQUENCY); + public static final Text FULL_CONTENT_TEXT = new Text(FULL_CONTENT); + + private ColumnFamilyConstants() { + // enforce static access + } +} diff --git a/core/tables/src/main/java/datawave/table/constants/LocalityGroupConstants.java b/core/tables/src/main/java/datawave/table/constants/LocalityGroupConstants.java new file mode 100644 index 00000000000..d44ea5d9f16 --- /dev/null +++ b/core/tables/src/main/java/datawave/table/constants/LocalityGroupConstants.java @@ -0,0 +1,15 @@ +package datawave.table.constants; + +/** + * Constants for locality group names reserved by DataWave + */ +public class LocalityGroupConstants { + + public static String FULL_CONTENT_LOCALITY = "fullcontent"; + + public static String TERM_FREQUENCY_LOCALITY = "termfrequency"; + + private LocalityGroupConstants() { + // enforce static access + } +} diff --git a/core/tables/src/test/java/datawave/table/constants/ColumnFamilyConstantsTest.java b/core/tables/src/test/java/datawave/table/constants/ColumnFamilyConstantsTest.java new file mode 100644 index 00000000000..066fd908ee2 --- /dev/null +++ b/core/tables/src/test/java/datawave/table/constants/ColumnFamilyConstantsTest.java @@ -0,0 +1,27 @@ +package datawave.table.constants; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.apache.hadoop.io.Text; +import org.junit.jupiter.api.Test; + +/** + * It may seem odd to have a unit test for a constant class. However, a failing unit test here is a hint that changes could affect child modules in an + * unexpected way. + *

+ * Thus, great care should be taken when modifying core constants. + */ +public class ColumnFamilyConstantsTest { + + @Test + public void testConstantsAsString() { + assertEquals("tf", ColumnFamilyConstants.TERM_FREQUENCY); + assertEquals("d", ColumnFamilyConstants.FULL_CONTENT); + } + + @Test + public void testConstantsAsText() { + assertEquals(new Text("tf"), ColumnFamilyConstants.TERM_FREQUENCY_TEXT); + assertEquals(new Text("d"), ColumnFamilyConstants.FULL_CONTENT_TEXT); + } +} diff --git a/core/tables/src/test/java/datawave/table/constants/LocalityGroupConstantsTest.java b/core/tables/src/test/java/datawave/table/constants/LocalityGroupConstantsTest.java new file mode 100644 index 00000000000..f47c3668976 --- /dev/null +++ b/core/tables/src/test/java/datawave/table/constants/LocalityGroupConstantsTest.java @@ -0,0 +1,19 @@ +package datawave.table.constants; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +/** + * These constants should not change very often, if at all. + *

+ * In the event that these need to change please ensure all child modules are updated + */ +public class LocalityGroupConstantsTest { + + @Test + public void testLocalityGroupNames() { + assertEquals("fullcontent", LocalityGroupConstants.FULL_CONTENT_LOCALITY); + assertEquals("termfrequency", LocalityGroupConstants.TERM_FREQUENCY_LOCALITY); + } +} diff --git a/warehouse/ingest-core/pom.xml b/warehouse/ingest-core/pom.xml index e3ab28cb574..51c8aa962b9 100644 --- a/warehouse/ingest-core/pom.xml +++ b/warehouse/ingest-core/pom.xml @@ -46,6 +46,10 @@ datawave-core-common-util ${project.version} + + gov.nsa.datawave.core + datawave-core-tables + io.dropwizard.metrics metrics-core diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/ExtendedDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/ExtendedDataTypeHandler.java index a25f1368f70..0285e163d4f 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/ExtendedDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/ExtendedDataTypeHandler.java @@ -24,10 +24,18 @@ public interface ExtendedDataTypeHandler extends DataTypeHandler { Value NULL_VALUE = new Value(new byte[0]); + + @Deprecated(forRemoval = true, since = "7.40.0") String FULL_CONTENT_LOCALITY_NAME = "fullcontent"; + + @Deprecated(forRemoval = true, since = "7.40.0") String FULL_CONTENT_COLUMN_FAMILY = "d"; /* TODO Make a clearer definition of full content indexers */ + + @Deprecated(forRemoval = true, since = "7.40.0") String TERM_FREQUENCY_LOCALITY_NAME = "termfrequency"; + + @Deprecated(forRemoval = true, since = "7.40.0") Text TERM_FREQUENCY_COLUMN_FAMILY = new Text("tf"); long process(KEYIN key, RawRecordContainer event, Multimap fields, diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java index 6f416d02fff..038c9ccfe66 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java @@ -42,6 +42,7 @@ import datawave.ingest.mapreduce.job.BulkIngestKey; import datawave.ingest.mapreduce.job.writer.ContextWriter; import datawave.marking.MarkingFunctions; +import datawave.table.constants.ColumnFamilyConstants; /** * Handler that take events with processing errors or fatal errors and dumps them into a processing error table. This table will be used for subsequent @@ -311,8 +312,8 @@ record = record.copy(); // ShardId 'd' DataType\0UID\0Name for document content event using Event.Writable String colq = record.getDataType().outputName() + '\0' + record.getId() + '\0' + EVENT_CONTENT_FIELD; - Key k = createKey(getShardId(record), new Text(ExtendedDataTypeHandler.FULL_CONTENT_COLUMN_FAMILY), new Text(colq), getVisibility(record, null), - record.getTimestamp(), this.helper.getDeleteMode()); + Key k = createKey(getShardId(record), new Text(ColumnFamilyConstants.FULL_CONTENT), new Text(colq), getVisibility(record, null), record.getTimestamp(), + this.helper.getDeleteMode()); BulkIngestKey ebKey = new BulkIngestKey(getShardTableName(), k); contextWriter.write(ebKey, value, context); diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/tokenize/ContentIndexingColumnBasedHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/tokenize/ContentIndexingColumnBasedHandler.java index ba8304fd36c..075ecc1bd52 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/tokenize/ContentIndexingColumnBasedHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/tokenize/ContentIndexingColumnBasedHandler.java @@ -38,7 +38,6 @@ import datawave.ingest.data.tokenize.TokenizationHelper.HeartBeatThread; import datawave.ingest.data.tokenize.TokenizationHelper.TokenizerTimeoutException; import datawave.ingest.data.tokenize.TruncateAttribute; -import datawave.ingest.mapreduce.handler.ExtendedDataTypeHandler; import datawave.ingest.mapreduce.handler.shard.AbstractColumnBasedHandler; import datawave.ingest.mapreduce.handler.shard.ShardedDataTypeHandler; import datawave.ingest.mapreduce.handler.shard.content.BoundedOffsetQueue; @@ -52,6 +51,7 @@ import datawave.ingest.util.BloomFilterWrapper; import datawave.ingest.util.Identity; import datawave.ingest.util.TimeoutStrategy; +import datawave.table.constants.ColumnFamilyConstants; import datawave.util.TextUtil; /** @@ -603,9 +603,8 @@ protected void createTermFrequencyIndex(RawRecordContainer event, Multimap context, StatusReporter reporter, Text uid, byte[] visibility, byte[] shardId, byte[] rawValue) throws IOException, InterruptedException, MutationsRejectedException { - Key k = createKey(shardId, new Text(ExtendedDataTypeHandler.FULL_CONTENT_COLUMN_FAMILY), uid, visibility, event.getTimestamp(), - this.ingestHelper.getDeleteMode()); + Key k = createKey(shardId, new Text(ColumnFamilyConstants.FULL_CONTENT), uid, visibility, event.getTimestamp(), this.ingestHelper.getDeleteMode()); ByteArrayOutputStream baos = null; Base64OutputStream b64os = null; @@ -805,9 +805,8 @@ protected void createTermFrequencyIndex(RawRecordContainer event, ContextWriter< colq.append(this.eventDataTypeName).append('\u0000').append(this.eventUid).append('\u0000').append(nfv.getIndexedFieldValue()).append('\u0000') .append(nfv.getIndexedFieldName()); - BulkIngestKey bKey = new BulkIngestKey(new Text(this.getShardTableName()), - new Key(shardId, ExtendedDataTypeHandler.TERM_FREQUENCY_COLUMN_FAMILY.getBytes(), colq.toString().getBytes(), visibility, - event.getTimestamp(), deleteMode)); + BulkIngestKey bKey = new BulkIngestKey(new Text(this.getShardTableName()), new Key(shardId, ColumnFamilyConstants.TERM_FREQUENCY_TEXT.getBytes(), + colq.toString().getBytes(), visibility, event.getTimestamp(), deleteMode)); contextWriter.write(bKey, value, context); } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/DateIndexTableConfigHelper.java b/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/DateIndexTableConfigHelper.java index 0b1db2e7c3a..4176b83ab4a 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/DateIndexTableConfigHelper.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/DateIndexTableConfigHelper.java @@ -15,7 +15,8 @@ import org.apache.hadoop.util.StringUtils; import org.apache.log4j.Logger; -import datawave.ingest.mapreduce.handler.ExtendedDataTypeHandler; +import datawave.table.constants.ColumnFamilyConstants; +import datawave.table.constants.LocalityGroupConstants; public class DateIndexTableConfigHelper extends AbstractTableConfigHelper { @@ -38,9 +39,8 @@ public void setup(String tableName, Configuration config, Logger log) throws Ill } String localityGroupsConf = null; - localityGroupsConf = conf.get(LOCALITY_GROUPS, - ExtendedDataTypeHandler.FULL_CONTENT_LOCALITY_NAME + ':' + ExtendedDataTypeHandler.FULL_CONTENT_COLUMN_FAMILY + ',' - + ExtendedDataTypeHandler.TERM_FREQUENCY_LOCALITY_NAME + ':' + ExtendedDataTypeHandler.TERM_FREQUENCY_COLUMN_FAMILY); + localityGroupsConf = conf.get(LOCALITY_GROUPS, LocalityGroupConstants.FULL_CONTENT_LOCALITY + ':' + ColumnFamilyConstants.FULL_CONTENT + ',' + + LocalityGroupConstants.TERM_FREQUENCY_LOCALITY + ':' + ColumnFamilyConstants.TERM_FREQUENCY); for (String localityGroupDefConf : StringUtils.split(localityGroupsConf)) { String[] localityGroupDef = StringUtils.split(localityGroupDefConf, '\\', ':'); Set families = localityGroups.get(localityGroupDef[0]); diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/ErrorShardTableConfigHelper.java b/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/ErrorShardTableConfigHelper.java index 2cc2f6b51fd..f408f5b8bd4 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/ErrorShardTableConfigHelper.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/ErrorShardTableConfigHelper.java @@ -8,9 +8,10 @@ import org.apache.hadoop.util.StringUtils; import org.apache.log4j.Logger; -import datawave.ingest.mapreduce.handler.ExtendedDataTypeHandler; import datawave.ingest.mapreduce.handler.error.ErrorShardedDataTypeHandler; import datawave.ingest.mapreduce.handler.shard.ShardedDataTypeHandler; +import datawave.table.constants.ColumnFamilyConstants; +import datawave.table.constants.LocalityGroupConstants; /** * TableConfigHelper implementation for the "sharded" error tables. This class should perform the majority of the same operations that the @@ -44,9 +45,8 @@ public void setup(String tableName, Configuration config, Logger log) throws Ill String localityGroupsConf = null; if (tableName.equals(shardTableName)) { localityGroupsConf = conf.get(shardTableName + LOCALITY_GROUPS, - ExtendedDataTypeHandler.FULL_CONTENT_LOCALITY_NAME + ':' + ExtendedDataTypeHandler.FULL_CONTENT_COLUMN_FAMILY + ',' - + ExtendedDataTypeHandler.TERM_FREQUENCY_LOCALITY_NAME + ':' - + ExtendedDataTypeHandler.TERM_FREQUENCY_COLUMN_FAMILY); + LocalityGroupConstants.FULL_CONTENT_LOCALITY + ':' + ColumnFamilyConstants.FULL_CONTENT + ',' + + LocalityGroupConstants.TERM_FREQUENCY_LOCALITY + ':' + ColumnFamilyConstants.TERM_FREQUENCY); for (String localityGroupDefConf : StringUtils.split(localityGroupsConf)) { String[] localityGroupDef = StringUtils.split(localityGroupDefConf, '\\', ':'); Set families = localityGroups.get(localityGroupDef[0]); diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/ShardTableConfigHelper.java b/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/ShardTableConfigHelper.java index b4225b37cbc..506862bb4d1 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/ShardTableConfigHelper.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/ShardTableConfigHelper.java @@ -18,7 +18,6 @@ import org.apache.hadoop.util.StringUtils; import org.apache.log4j.Logger; -import datawave.ingest.mapreduce.handler.ExtendedDataTypeHandler; import datawave.ingest.mapreduce.handler.shard.ShardedDataTypeHandler; import datawave.ingest.table.aggregator.BitSetCombiner; import datawave.ingest.table.aggregator.CombinerConfiguration; @@ -28,6 +27,8 @@ import datawave.ingest.table.balancer.ShardedTableTabletBalancer; import datawave.ingest.table.bloomfilter.ShardIndexKeyFunctor; import datawave.ingest.table.bloomfilter.ShardKeyFunctor; +import datawave.table.constants.ColumnFamilyConstants; +import datawave.table.constants.LocalityGroupConstants; import datawave.util.TableName; public class ShardTableConfigHelper extends AbstractTableConfigHelper { @@ -106,9 +107,8 @@ public void setup(String tableName, Configuration config, Logger log) throws Ill String localityGroupsConf = null; if (tableName.equals(shardTableName)) { localityGroupsConf = conf.get(shardTableName + LOCALITY_GROUPS, - ExtendedDataTypeHandler.FULL_CONTENT_LOCALITY_NAME + ':' + ExtendedDataTypeHandler.FULL_CONTENT_COLUMN_FAMILY + ',' - + ExtendedDataTypeHandler.TERM_FREQUENCY_LOCALITY_NAME + ':' - + ExtendedDataTypeHandler.TERM_FREQUENCY_COLUMN_FAMILY); + LocalityGroupConstants.FULL_CONTENT_LOCALITY + ':' + ColumnFamilyConstants.FULL_CONTENT + ',' + + LocalityGroupConstants.TERM_FREQUENCY_LOCALITY + ':' + ColumnFamilyConstants.TERM_FREQUENCY); for (String localityGroupDefConf : StringUtils.split(localityGroupsConf)) { String[] localityGroupDef = StringUtils.split(localityGroupDefConf, '\\', ':'); Set families = localityGroups.get(localityGroupDef[0]); diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/edge/EdgeHandlerTestUtil.java b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/edge/EdgeHandlerTestUtil.java index 31273c01749..c44e6d615b3 100644 --- a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/edge/EdgeHandlerTestUtil.java +++ b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/edge/EdgeHandlerTestUtil.java @@ -28,6 +28,7 @@ import datawave.ingest.mapreduce.job.writer.AbstractContextWriter; import datawave.ingest.test.StandaloneStatusReporter; import datawave.ingest.test.StandaloneTaskAttemptContext; +import datawave.table.constants.ColumnFamilyConstants; import datawave.util.TableName; public class EdgeHandlerTestUtil { @@ -43,7 +44,7 @@ public class EdgeHandlerTestUtil { private static Logger log = Logger.getLogger(EdgeHandlerTestUtil.class); public static boolean isDocumentKey(Key k) { - return isShardKey(k) && k.getColumnFamily().toString().equals(ExtendedDataTypeHandler.FULL_CONTENT_COLUMN_FAMILY); + return isShardKey(k) && k.getColumnFamily().toString().equals(ColumnFamilyConstants.FULL_CONTENT); } public static boolean isShardKey(Key k) { diff --git a/warehouse/ingest-csv/src/test/java/datawave/ingest/csv/mr/handler/ColumnBasedHandlerTestUtil.java b/warehouse/ingest-csv/src/test/java/datawave/ingest/csv/mr/handler/ColumnBasedHandlerTestUtil.java index f65d7461246..df926050b3c 100644 --- a/warehouse/ingest-csv/src/test/java/datawave/ingest/csv/mr/handler/ColumnBasedHandlerTestUtil.java +++ b/warehouse/ingest-csv/src/test/java/datawave/ingest/csv/mr/handler/ColumnBasedHandlerTestUtil.java @@ -34,6 +34,7 @@ import datawave.ingest.mapreduce.job.writer.AbstractContextWriter; import datawave.ingest.test.StandaloneStatusReporter; import datawave.ingest.test.StandaloneTaskAttemptContext; +import datawave.table.constants.ColumnFamilyConstants; import datawave.util.TableName; /** @@ -52,7 +53,7 @@ public class ColumnBasedHandlerTestUtil { private static final Logger log = Logger.getLogger(ColumnBasedHandlerTestUtil.class); public static boolean isDocumentKey(Key k) { - return isShardKey(k) && k.getColumnFamily().toString().equals(ExtendedDataTypeHandler.FULL_CONTENT_COLUMN_FAMILY); + return isShardKey(k) && k.getColumnFamily().toString().equals(ColumnFamilyConstants.FULL_CONTENT); } public static boolean isShardKey(Key k) { diff --git a/warehouse/ingest-json/src/test/java/datawave/ingest/json/mr/handler/ColumnBasedHandlerTestUtil.java b/warehouse/ingest-json/src/test/java/datawave/ingest/json/mr/handler/ColumnBasedHandlerTestUtil.java index 6861d9af054..6f5431a0030 100644 --- a/warehouse/ingest-json/src/test/java/datawave/ingest/json/mr/handler/ColumnBasedHandlerTestUtil.java +++ b/warehouse/ingest-json/src/test/java/datawave/ingest/json/mr/handler/ColumnBasedHandlerTestUtil.java @@ -33,6 +33,7 @@ import datawave.ingest.mapreduce.job.writer.AbstractContextWriter; import datawave.ingest.test.StandaloneStatusReporter; import datawave.ingest.test.StandaloneTaskAttemptContext; +import datawave.table.constants.ColumnFamilyConstants; import datawave.util.TableName; /** @@ -50,7 +51,7 @@ public class ColumnBasedHandlerTestUtil { private static Logger log = Logger.getLogger(ColumnBasedHandlerTestUtil.class); public static boolean isDocumentKey(Key k) { - return isShardKey(k) && k.getColumnFamily().toString().equals(ExtendedDataTypeHandler.FULL_CONTENT_COLUMN_FAMILY); + return isShardKey(k) && k.getColumnFamily().toString().equals(ColumnFamilyConstants.FULL_CONTENT); } public static boolean isShardKey(Key k) { diff --git a/warehouse/pom.xml b/warehouse/pom.xml index 30dd8147ca2..0710e78c91c 100644 --- a/warehouse/pom.xml +++ b/warehouse/pom.xml @@ -148,6 +148,11 @@ datawave-regression-testing ${project.version} + + gov.nsa.datawave.core + datawave-core-tables + ${project.version} + gov.nsa.datawave.webservices datawave-ws-annotations diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/content/ContentQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/content/ContentQueryLogic.java index 0e4087ff0fe..339f545ee9b 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/content/ContentQueryLogic.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/content/ContentQueryLogic.java @@ -29,7 +29,6 @@ import datawave.core.query.logic.QueryCheckpoint; import datawave.core.query.logic.QueryKey; import datawave.core.query.logic.QueryLogicTransformer; -import datawave.ingest.mapreduce.handler.ExtendedDataTypeHandler; import datawave.microservice.query.Query; import datawave.microservice.query.QueryImpl.Parameter; import datawave.query.Constants; @@ -37,6 +36,7 @@ import datawave.query.config.ContentQueryConfiguration; import datawave.query.tables.ScannerFactory; import datawave.query.transformer.ContentQueryTransformer; +import datawave.table.constants.ColumnFamilyConstants; import datawave.webservice.query.exception.QueryException; /** @@ -228,7 +228,7 @@ private Collection createRanges(final Query settings, final String endKey log.debug("Received pieces: " + shardId + ", " + datatype + ", " + uid); // Create and add a Range - final String cf = ExtendedDataTypeHandler.FULL_CONTENT_COLUMN_FAMILY; + final String cf = ColumnFamilyConstants.FULL_CONTENT; final String cq = datatype + Constants.NULL_BYTE_STRING + uid; final Key startKey = new Key(shardId, cf, cq + Constants.NULL_BYTE_STRING); final Key endKey = new Key(shardId, cf, cq + endKeyTerminator); diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/keyword/KeywordQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/keyword/KeywordQueryLogic.java index da551b54dd4..476da19ce60 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/keyword/KeywordQueryLogic.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/keyword/KeywordQueryLogic.java @@ -37,7 +37,6 @@ import datawave.core.query.logic.QueryCheckpoint; import datawave.core.query.logic.QueryKey; import datawave.core.query.logic.QueryLogicTransformer; -import datawave.ingest.mapreduce.handler.ExtendedDataTypeHandler; import datawave.microservice.query.Query; import datawave.microservice.query.QueryImpl.Parameter; import datawave.query.Constants; @@ -48,6 +47,7 @@ import datawave.query.tables.keyword.transform.KeywordResultsTransformer; import datawave.query.tables.keyword.transform.TagCloudInputTransformer; import datawave.query.transformer.TagCloudTransformer; +import datawave.table.constants.ColumnFamilyConstants; import datawave.util.keyword.TagCloudPartition; import datawave.util.keyword.TagCloudUtils; import datawave.webservice.query.exception.QueryException; @@ -387,7 +387,7 @@ private static Range getRangeFromTermParts(String[] parts, String endKeyTerminat log.debug("Received pieces: " + shardId + ", " + datatype + ", " + uid); // Create and add a Range - final String cf = ExtendedDataTypeHandler.FULL_CONTENT_COLUMN_FAMILY; + final String cf = ColumnFamilyConstants.FULL_CONTENT; final String cq = datatype + Constants.NULL_BYTE_STRING + uid; final Key startKey = new Key(shardId, cf, cq + Constants.NULL_BYTE_STRING); final Key endKey = new Key(shardId, cf, cq + endKeyTerminator); diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/term/TermFrequencyQueryTable.java b/warehouse/query-core/src/main/java/datawave/query/tables/term/TermFrequencyQueryTable.java index 2d795e6aa84..50c2b8bffab 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/term/TermFrequencyQueryTable.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/term/TermFrequencyQueryTable.java @@ -22,13 +22,13 @@ import datawave.core.query.configuration.GenericQueryConfiguration; import datawave.core.query.logic.BaseQueryLogic; import datawave.core.query.logic.QueryLogicTransformer; -import datawave.ingest.mapreduce.handler.ExtendedDataTypeHandler; import datawave.microservice.query.Query; import datawave.microservice.query.QueryImpl.Parameter; import datawave.query.QueryParameters; import datawave.query.config.TermFrequencyQueryConfiguration; import datawave.query.transformer.TermFrequencyQueryTransformer; import datawave.query.util.QueryScannerHelper; +import datawave.table.constants.ColumnFamilyConstants; import datawave.webservice.query.exception.QueryException; public class TermFrequencyQueryTable extends BaseQueryLogic> { @@ -112,7 +112,7 @@ public GenericQueryConfiguration initialize(AccumuloClient client, Query setting END = ALL; } - final String tf = ExtendedDataTypeHandler.TERM_FREQUENCY_COLUMN_FAMILY.toString(); + final String tf = ColumnFamilyConstants.TERM_FREQUENCY_TEXT.toString(); Key startKey = new Key(shardId, tf, datatype + NULL + uid + NULL); Key endKey = new Key(shardId, tf, datatype + NULL + uid + END); Range r = new Range(startKey, true, endKey, false); diff --git a/warehouse/query-core/src/test/java/datawave/query/iterator/logic/ContentSummaryIteratorTest.java b/warehouse/query-core/src/test/java/datawave/query/iterator/logic/ContentSummaryIteratorTest.java index cf427bbc260..b7d33b5f66d 100644 --- a/warehouse/query-core/src/test/java/datawave/query/iterator/logic/ContentSummaryIteratorTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/iterator/logic/ContentSummaryIteratorTest.java @@ -35,15 +35,15 @@ import org.junit.Test; import org.junit.runner.RunWith; -import datawave.ingest.mapreduce.handler.ExtendedDataTypeHandler; import datawave.query.Constants; import datawave.query.iterator.SortedListKeyValueIterator; +import datawave.table.constants.ColumnFamilyConstants; @RunWith(EasyMockRunner.class) public class ContentSummaryIteratorTest extends EasyMockSupport { private static final Text row = new Text("20220115_1"); - private static final Text colf = new Text(ExtendedDataTypeHandler.FULL_CONTENT_COLUMN_FAMILY); + private static final Text colf = new Text(ColumnFamilyConstants.FULL_CONTENT); @Mock private IteratorEnvironment env; diff --git a/warehouse/query-core/src/test/java/datawave/query/iterator/logic/TermFrequencyExcerptIteratorTest.java b/warehouse/query-core/src/test/java/datawave/query/iterator/logic/TermFrequencyExcerptIteratorTest.java index 0cd801341f7..20942ac8bbe 100644 --- a/warehouse/query-core/src/test/java/datawave/query/iterator/logic/TermFrequencyExcerptIteratorTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/iterator/logic/TermFrequencyExcerptIteratorTest.java @@ -36,16 +36,16 @@ import com.google.common.collect.Multimap; import datawave.ingest.data.config.NormalizedFieldAndValue; -import datawave.ingest.mapreduce.handler.ExtendedDataTypeHandler; import datawave.ingest.protobuf.TermWeight; import datawave.query.Constants; import datawave.query.iterator.SortedListKeyValueIterator; +import datawave.table.constants.ColumnFamilyConstants; @RunWith(EasyMockRunner.class) public class TermFrequencyExcerptIteratorTest extends EasyMockSupport { private static final Text row = new Text("20220115_1"); - private static final Text colf = ExtendedDataTypeHandler.TERM_FREQUENCY_COLUMN_FAMILY; + private static final Text colf = ColumnFamilyConstants.TERM_FREQUENCY_TEXT; @Mock private IteratorEnvironment env; diff --git a/warehouse/query-core/src/test/java/datawave/query/jexl/functions/ContentFunctionQueryTest.java b/warehouse/query-core/src/test/java/datawave/query/jexl/functions/ContentFunctionQueryTest.java index 6bbe523f40a..6fb779be079 100644 --- a/warehouse/query-core/src/test/java/datawave/query/jexl/functions/ContentFunctionQueryTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/jexl/functions/ContentFunctionQueryTest.java @@ -71,7 +71,6 @@ import datawave.ingest.data.config.ingest.AbstractContentIngestHelper; import datawave.ingest.data.config.ingest.ContentBaseIngestHelper; import datawave.ingest.data.config.ingest.TermFrequencyIngestHelperInterface; -import datawave.ingest.mapreduce.handler.ExtendedDataTypeHandler; import datawave.ingest.mapreduce.handler.dateindex.DateIndexDataTypeHandler; import datawave.ingest.mapreduce.handler.shard.ShardedDataTypeHandler; import datawave.ingest.mapreduce.handler.tokenize.ContentIndexingColumnBasedHandler; @@ -94,6 +93,7 @@ import datawave.query.tables.ShardQueryLogic; import datawave.query.tables.edge.DefaultEdgeEventQueryLogic; import datawave.query.testframework.MockStatusReporter; +import datawave.table.constants.ColumnFamilyConstants; import datawave.util.TableName; import datawave.webservice.edgedictionary.RemoteEdgeDictionary; import datawave.webservice.query.result.event.DefaultEvent; @@ -455,9 +455,8 @@ private void getTFKey(final NormalizedFieldAndValue nfv, final RawRecordContaine colq.append(this.eventDataTypeName).append('\u0000').append(this.eventUid).append('\u0000').append(nfv.getIndexedFieldValue()).append('\u0000') .append(nfv.getIndexedFieldName()); - BulkIngestKey bKey = new BulkIngestKey(new Text(this.getShardTableName()), - new Key(shardId, ExtendedDataTypeHandler.TERM_FREQUENCY_COLUMN_FAMILY.getBytes(), colq.toString().getBytes(), fieldVisibility, - event.getDate(), helper.getDeleteMode())); + BulkIngestKey bKey = new BulkIngestKey(new Text(this.getShardTableName()), new Key(shardId, ColumnFamilyConstants.TERM_FREQUENCY_TEXT.getBytes(), + colq.toString().getBytes(), fieldVisibility, event.getDate(), helper.getDeleteMode())); values.put(bKey, new Value(info.toByteArray())); }