From a784c145ea8d7ed61a7be24b38fe0327a5709405 Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Mon, 16 Mar 2026 13:59:18 -0400 Subject: [PATCH 01/63] updated accumulo-utils to use accumulo-access --- core/utils/accumulo-utils/pom.xml | 20 ++- .../marking/AccessExpressionMarkings.java | 18 +++ .../marking/ColumnVisibilityCache.java | 22 --- .../marking/ColumnVisibilityHelper.java | 133 ------------------ .../marking/FlattenedVisibilityCache.java | 41 ------ .../datawave/marking/MarkingFunctions.java | 87 +++++------- .../main/java/datawave/marking/Markings.java | 7 + .../datawave/marking/VisibilityFlattener.java | 46 ------ .../MarkingFunctionsDefaultCombineTest.java | 50 +++---- 9 files changed, 94 insertions(+), 330 deletions(-) create mode 100644 core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java delete mode 100644 core/utils/accumulo-utils/src/main/java/datawave/marking/ColumnVisibilityCache.java delete mode 100644 core/utils/accumulo-utils/src/main/java/datawave/marking/ColumnVisibilityHelper.java delete mode 100644 core/utils/accumulo-utils/src/main/java/datawave/marking/FlattenedVisibilityCache.java create mode 100644 core/utils/accumulo-utils/src/main/java/datawave/marking/Markings.java delete mode 100644 core/utils/accumulo-utils/src/main/java/datawave/marking/VisibilityFlattener.java diff --git a/core/utils/accumulo-utils/pom.xml b/core/utils/accumulo-utils/pom.xml index 5dfe15b44b9..fa2a2610d63 100644 --- a/core/utils/accumulo-utils/pom.xml +++ b/core/utils/accumulo-utils/pom.xml @@ -64,13 +64,13 @@ org.apache.accumulo - accumulo-core - - - org.slf4j - slf4j-api - - + accumulo-access-core + 1.0.0-java-11 + + + org.apache.accumulo + accumulo-access-examples + 1.0.0-java-11 org.apache.commons @@ -93,6 +93,12 @@ org.xerial.snappy snappy-java + + org.projectlombok + lombok + 1.18.36 + provided + org.springframework spring-jcl diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java new file mode 100644 index 00000000000..b924cf0018a --- /dev/null +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java @@ -0,0 +1,18 @@ +package datawave.marking; + +import org.apache.accumulo.access.AccessExpression; + +import lombok.Builder; +import lombok.Data; + +@Data +@Builder +public class AccessExpressionMarkings implements Markings { + + private final AccessExpression accessExpression; + + @Override + public AccessExpression getMarkings() { + return getAccessExpression(); + } +} diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/ColumnVisibilityCache.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/ColumnVisibilityCache.java deleted file mode 100644 index 27fe70ebdfb..00000000000 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/ColumnVisibilityCache.java +++ /dev/null @@ -1,22 +0,0 @@ -package datawave.marking; - -import java.util.Collections; -import java.util.Map; - -import org.apache.accumulo.core.data.ByteSequence; -import org.apache.accumulo.core.security.ColumnVisibility; -import org.apache.commons.collections4.map.LRUMap; - -public class ColumnVisibilityCache { - @SuppressWarnings("unchecked") - private static Map cache = Collections.synchronizedMap(new LRUMap(256)); - - public static ColumnVisibility get(ByteSequence bytes) { - ColumnVisibility vis = cache.get(bytes); - if (vis == null) { - vis = new ColumnVisibility(bytes.toArray()); - cache.put(bytes, vis); - } - return vis; - } -} diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/ColumnVisibilityHelper.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/ColumnVisibilityHelper.java deleted file mode 100644 index 7170eb64428..00000000000 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/ColumnVisibilityHelper.java +++ /dev/null @@ -1,133 +0,0 @@ -package datawave.marking; - -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import org.apache.accumulo.core.security.Authorizations; -import org.apache.accumulo.core.security.ColumnVisibility; -import org.apache.accumulo.core.security.ColumnVisibility.Node; -import org.apache.accumulo.core.security.ColumnVisibility.NodeType; -import org.apache.accumulo.core.security.VisibilityEvaluator; -import org.apache.accumulo.core.security.VisibilityParseException; -import org.apache.log4j.Logger; - -import datawave.marking.MarkingFunctions.Exception; - -public class ColumnVisibilityHelper { - - protected static final Charset charset = StandardCharsets.UTF_8; - private static Logger log = Logger.getLogger(ColumnVisibilityHelper.class); - - public static ColumnVisibility simplifyColumnVisibilityForAuthorizations(ColumnVisibility columnVisibility, Collection authorizations) - throws MarkingFunctions.Exception { - - ColumnVisibility simplifiedCV = columnVisibility; - - Node node = columnVisibility.getParseTree(); - if (node.getType() == NodeType.OR) { - if (log.isTraceEnabled()) { - log.trace("Top level OR Node, removing unsatisfied branches from: " + columnVisibility); - } - byte[] expression = columnVisibility.getExpression(); - if (authorizations != null) { - HashSet ve = new HashSet<>(); - for (Authorizations a : authorizations) - ve.add(new VisibilityEvaluator(a)); - removeUnsatisfiedTopLevelOrNodes(ve, expression, node); - } - - simplifiedCV = ColumnVisibilityHelper.flatten(node, expression); - if (log.isTraceEnabled()) { - log.trace("removed unsatisfied branches, visibility now: " + simplifiedCV); - } - } - return simplifiedCV; - } - - public static ColumnVisibility removeUndisplayedVisibilities(ColumnVisibility columnVisibility, Set undisplayedVisibilities) - throws MarkingFunctions.Exception { - ColumnVisibility newColumnVisibility = columnVisibility; - if (undisplayedVisibilities != null && !undisplayedVisibilities.isEmpty()) { - byte[] expression = columnVisibility.getExpression(); - Node node = columnVisibility.getParseTree(); - removeUndisplayedVisibilities(node, expression, undisplayedVisibilities); - newColumnVisibility = flatten(node, expression); - } - return newColumnVisibility; - } - - private static void removeUnsatisfiedTopLevelOrNodes(Set ve, byte[] expression, Node node) throws Exception { - if (node.getType() == NodeType.OR) { - List children = node.getChildren(); - int lastNode = children.size() - 1; - for (int x = lastNode; x >= 0; x--) { - Node currNode = children.get(x); - boolean remove = isUnsatisfied(x, ve, currNode, expression); - if (remove == true) { - children.remove(x); - } - } - } - } - - private static boolean isUnsatisfied(int position, Set ve, Node currNode, byte[] expression) throws Exception { - boolean unsatisfied = false; - try { - ColumnVisibility currVis = ColumnVisibilityHelper.flatten(currNode, expression); - for (VisibilityEvaluator v : ve) { - if (!v.evaluate(currVis)) - unsatisfied = true; - } - } catch (VisibilityParseException e) { - throw new MarkingFunctions.Exception(e); - } - return unsatisfied; - } - - private static String termNodeToString(Node termNode, byte[] expression) throws Exception { - int start = termNode.getTermStart(); - int end = termNode.getTermEnd(); - - String str = "[ERROR]"; - try { - str = new String(expression, start, end - start, charset); - } catch (RuntimeException e) { - log.error("Error converting term: start:" + start + " length:" + (end - start) + "of expression:" + expression + " -- " + e.getMessage()); - throw new MarkingFunctions.Exception(e); - } - return str; - } - - private static ColumnVisibility flatten(Node node, byte[] expression) { - - Node newNode = ColumnVisibility.normalize(node, expression); - StringBuilder sb = new StringBuilder(); - ColumnVisibility.stringify(newNode, expression, sb); - return new ColumnVisibility(sb.toString()); - } - - private static void removeUndisplayedVisibilities(Node node, byte[] expression, Set undisplayedVisibilities) throws MarkingFunctions.Exception { - List children = node.getChildren(); - // walk backwards so we don't change the index that we need to remove - int lastNode = children.size() - 1; - for (int x = lastNode; x >= 0; x--) { - Node currNode = children.get(x); - if (currNode.getType() == NodeType.TERM) { - String nodeString = termNodeToString(currNode, expression); - if (undisplayedVisibilities.contains(nodeString)) { - children.remove(x); - } - } else { - removeUndisplayedVisibilities(currNode, expression, undisplayedVisibilities); - // if all children of this NodeType.AND or NodeType.OR node have been removed, remove the node itself. - if (currNode.getChildren().isEmpty()) { - children.remove(x); - } - } - } - } -} diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/FlattenedVisibilityCache.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/FlattenedVisibilityCache.java deleted file mode 100644 index 85e1697cc26..00000000000 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/FlattenedVisibilityCache.java +++ /dev/null @@ -1,41 +0,0 @@ -package datawave.marking; - -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -import org.apache.accumulo.core.data.ByteSequence; -import org.apache.accumulo.core.security.ColumnVisibility; - -/** - * This is a cache that can be used per process to save flattened visibility calculations. - * - */ -public class FlattenedVisibilityCache { - private static Map flattenedVisCache = Collections.synchronizedMap(new HashMap<>()); - - /** - * Create a flattened visibility, using the cache if possible - * - * @param vis - * the visibility to flatten - * @return the flattened visibility - */ - public static byte[] flatten(ColumnVisibility vis) { - byte[] visBytes = flattenedVisCache.get(vis); - if (visBytes == null) { - visBytes = vis.flatten(); - flattenedVisCache.put(vis, visBytes); - } - return visBytes; - } - - public static byte[] flatten(ByteSequence bytes) { - return flatten(ColumnVisibilityCache.get(bytes)); - } - - public static boolean equals(ColumnVisibility left, ColumnVisibility right) { - return Arrays.equals(flatten(left), flatten(right)); - } -} diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java index 9026edc78df..4928cd7aa70 100644 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java @@ -1,7 +1,5 @@ package datawave.marking; -import static java.nio.charset.StandardCharsets.UTF_8; - import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.util.Arrays; @@ -10,8 +8,10 @@ import java.util.Map; import java.util.stream.Collectors; -import org.apache.accumulo.core.security.Authorizations; -import org.apache.accumulo.core.security.ColumnVisibility; +import org.apache.accumulo.access.Access; +import org.apache.accumulo.access.AccessExpression; +import org.apache.accumulo.access.ParsedAccessExpression; +import org.apache.accumulo.access.examples.ParseExamples; import org.apache.commons.beanutils.BeanUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -19,8 +19,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Charsets; -import com.google.common.collect.Maps; /** * Accumulo marks all data with a columnVisibility that declares and controls access. MarkingFunctions provide a pattern for mapping a user's preferred means of @@ -29,26 +27,16 @@ * human-readable marking like "Patient Privileged Information" */ -public interface MarkingFunctions { - - ColumnVisibility combine(Collection columnVisibilities) throws MarkingFunctions.Exception; - - @SuppressWarnings("unchecked") - Map combine(Map... markings) throws MarkingFunctions.Exception; - - ColumnVisibility translateToColumnVisibility(Map markings) throws MarkingFunctions.Exception; +public interface MarkingFunctions> { - Map translateFromColumnVisibility(ColumnVisibility columnVisibility) throws MarkingFunctions.Exception; + AccessExpression combine(Collection columnVisibilities) throws MarkingFunctions.Exception; - Map translateFromColumnVisibilityForAuths(ColumnVisibility columnVisibility, Collection authorizations) - throws MarkingFunctions.Exception; + T combine(T... markings) throws MarkingFunctions.Exception; - Map translateFromColumnVisibilityForAuths(ColumnVisibility columnVisibility, Authorizations authorizations) - throws MarkingFunctions.Exception; + AccessExpression translateToColumnVisibility(T markings) throws MarkingFunctions.Exception; - byte[] flatten(ColumnVisibility vis); + T translateFromColumnVisibility(AccessExpression columnVisibility) throws MarkingFunctions.Exception; - @SuppressWarnings("serial") class Exception extends java.lang.Exception { public Exception() { @@ -72,53 +60,44 @@ public Exception(Throwable cause) { } } - class Default implements MarkingFunctions { - public static final String COLUMN_VISIBILITY = "columnVisibility"; + class Default implements MarkingFunctions { + public static final Access ACCESS = Access.builder().build(); @Override - public ColumnVisibility combine(Collection expressions) { + public AccessExpression combine(Collection expressions) { - // filter out any empty expressions, then flatten each one (to de-dupe) and concatenate with '&' - // flatten the final combined ColumnVisibility and use that to make the ColumnVisibility to return - return new ColumnVisibility(new ColumnVisibility(expressions.stream().map(ColumnVisibility::flatten).filter(b -> b.length > 0) - .map(b -> "(" + new String(b, UTF_8) + ")").collect(Collectors.joining("&")).getBytes(UTF_8)).flatten()); + // filter out any empty expressions and concatenate with '&' + // @formatter:off + ParsedAccessExpression expression = ACCESS.newParsedExpression(expressions + .stream() + .map(AccessExpression::getExpression) + .filter(str -> !str.isEmpty()) + .map(str -> "(" + str + ")") + .collect(Collectors.joining("&")) + ); + // @formatter:on + + // normalize the Parsed Expression and then return a copy without the parse tree + // FIXME: Using a beta of the accumulo-access API so temporarily using the normalize function from the example code + return ACCESS.newExpression(ParseExamples.normalize(expression).expression); } @Override - @SafeVarargs - public final Map combine(Map... markings) { + public final AccessExpressionMarkings combine(AccessExpressionMarkings... markings) { // translate COLUMN_VISIBILITY values to ColumnVisibility, combine them and // return translated back to Map - return translateFromColumnVisibility(combine(Arrays.stream(markings).filter(m -> m.containsKey(COLUMN_VISIBILITY)) - .map(this::translateToColumnVisibility).collect(Collectors.toSet()))); - } - - @Override - public ColumnVisibility translateToColumnVisibility(Map markings) { - ColumnVisibility cv = new ColumnVisibility(markings.get(COLUMN_VISIBILITY)); - return new ColumnVisibility(cv.flatten()); - } - - @Override - public Map translateFromColumnVisibility(ColumnVisibility expression) { - Map markings = Maps.newHashMap(); - markings.put(COLUMN_VISIBILITY, new String(expression.getExpression(), Charsets.UTF_8)); - return markings; - } - - @Override - public Map translateFromColumnVisibilityForAuths(ColumnVisibility columnVisibility, Collection authorizations) { - return translateFromColumnVisibility(columnVisibility); + return AccessExpressionMarkings.builder().accessExpression(combine(Arrays.stream(markings).map(Markings::getMarkings).collect(Collectors.toSet()))) + .build(); } @Override - public Map translateFromColumnVisibilityForAuths(ColumnVisibility columnVisibility, Authorizations authorizations) { - return translateFromColumnVisibility(columnVisibility); + public AccessExpression translateToColumnVisibility(AccessExpressionMarkings markings) { + return markings.getMarkings(); } @Override - public byte[] flatten(ColumnVisibility vis) { - return FlattenedVisibilityCache.flatten(vis); + public AccessExpressionMarkings translateFromColumnVisibility(AccessExpression expression) { + return AccessExpressionMarkings.builder().accessExpression(expression).build(); } } diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/Markings.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/Markings.java new file mode 100644 index 00000000000..4b89ef4a413 --- /dev/null +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/Markings.java @@ -0,0 +1,7 @@ +package datawave.marking; + +public interface Markings { + + T getMarkings(); + +} diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/VisibilityFlattener.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/VisibilityFlattener.java deleted file mode 100644 index 63cec96d4e4..00000000000 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/VisibilityFlattener.java +++ /dev/null @@ -1,46 +0,0 @@ -package datawave.marking; - -import static java.nio.charset.StandardCharsets.UTF_8; - -import java.util.Arrays; - -import org.apache.accumulo.core.security.ColumnVisibility; -import org.apache.accumulo.core.security.ColumnVisibility.Node; -import org.apache.accumulo.core.security.ColumnVisibility.NodeComparator; -import org.apache.accumulo.core.security.ColumnVisibility.NodeType; -import org.apache.hadoop.io.Text; - -public class VisibilityFlattener { - public static ColumnVisibility flatten(Node root, byte[] expression, boolean sort) { - StringBuilder out = new StringBuilder(); - flatten(root, expression, out, sort); - return new ColumnVisibility(out.toString()); - } - - public static Text flattenToText(Node root, byte[] expression, boolean sort) { - StringBuilder out = new StringBuilder(); - flatten(root, expression, out, sort); - return new Text(out.toString()); - } - - private static void flatten(Node root, byte[] expression, StringBuilder out, boolean sort) { - if (root.getType() == NodeType.TERM) - out.append(new String(expression, root.getTermStart(), root.getTermEnd() - root.getTermStart(), UTF_8)); - else { - String sep = ""; - Node[] children = root.getChildren().toArray(new Node[] {}); - if (sort) - Arrays.sort(children, new NodeComparator(expression)); - for (Node c : children) { - out.append(sep); - boolean parens = (c.getType() != NodeType.TERM && root.getType() != c.getType()); - if (parens) - out.append("("); - flatten(c, expression, out, sort); - if (parens) - out.append(")"); - sep = root.getType() == NodeType.AND ? "&" : "|"; - } - } - } -} diff --git a/core/utils/accumulo-utils/src/test/java/datawave/marking/MarkingFunctionsDefaultCombineTest.java b/core/utils/accumulo-utils/src/test/java/datawave/marking/MarkingFunctionsDefaultCombineTest.java index 7b23d454635..f9867efa87c 100644 --- a/core/utils/accumulo-utils/src/test/java/datawave/marking/MarkingFunctionsDefaultCombineTest.java +++ b/core/utils/accumulo-utils/src/test/java/datawave/marking/MarkingFunctionsDefaultCombineTest.java @@ -1,61 +1,57 @@ package datawave.marking; -import static datawave.marking.MarkingFunctions.Default.COLUMN_VISIBILITY; import static org.junit.jupiter.api.Assertions.assertEquals; -import java.util.Map; - -import org.apache.accumulo.core.security.ColumnVisibility; +import org.apache.accumulo.access.Access; +import org.apache.accumulo.access.AccessExpression; import org.junit.jupiter.api.Test; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; public class MarkingFunctionsDefaultCombineTest { + public static final Access ACCESS = Access.builder().build(); @Test public void testCombineAnds() throws MarkingFunctions.Exception { - MarkingFunctions markingFunctions = new MarkingFunctions.Default(); + MarkingFunctions markingFunctions = new MarkingFunctions.Default(); - ColumnVisibility oneAnna = new ColumnVisibility("A&B"); - ColumnVisibility twoAnna = new ColumnVisibility("A&C"); + AccessExpression oneAnna = ACCESS.newExpression("A&B"); + AccessExpression twoAnna = ACCESS.newExpression("A&C"); - ColumnVisibility combined = markingFunctions.combine(Sets.newHashSet(oneAnna, twoAnna)); - assertEquals(new ColumnVisibility("A&B&C"), combined); + AccessExpression combined = markingFunctions.combine(Sets.newHashSet(oneAnna, twoAnna)); + assertEquals(ACCESS.newExpression("A&B&C"), combined); } @Test public void testCombineOrs() throws MarkingFunctions.Exception { - MarkingFunctions markingFunctions = new MarkingFunctions.Default(); + MarkingFunctions markingFunctions = new MarkingFunctions.Default(); - ColumnVisibility oneOr = new ColumnVisibility("A|B"); - ColumnVisibility twoOr = new ColumnVisibility("A|C"); + AccessExpression oneOr = ACCESS.newExpression("A|B"); + AccessExpression twoOr = ACCESS.newExpression("A|C"); - ColumnVisibility combined = markingFunctions.combine(Sets.newHashSet(oneOr, twoOr)); - assertEquals(new ColumnVisibility("(A|B)&(A|C)"), combined); + AccessExpression combined = markingFunctions.combine(Sets.newHashSet(oneOr, twoOr)); + assertEquals(ACCESS.newExpression("(A|B)&(A|C)"), combined); } @Test public void testCombineMapsOfAnds() throws MarkingFunctions.Exception { - MarkingFunctions markingFunctions = new MarkingFunctions.Default(); - - Map mapOne = ImmutableMap.of(COLUMN_VISIBILITY, "A&B"); - Map mapTwo = ImmutableMap.of(COLUMN_VISIBILITY, "A&C"); + MarkingFunctions markingFunctions = new MarkingFunctions.Default(); - Map expected = ImmutableMap.of(COLUMN_VISIBILITY, "A&B&C"); + AccessExpressionMarkings cv1 = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression("A&B")).build(); + AccessExpressionMarkings cv2 = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression("A&C")).build(); + AccessExpressionMarkings expected = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression("A&B&C")).build(); - assertEquals(expected, markingFunctions.combine(mapOne, mapTwo)); + assertEquals(expected, markingFunctions.combine(cv1, cv2)); } @Test public void testCombineMapsOfOrs() throws MarkingFunctions.Exception { - MarkingFunctions markingFunctions = new MarkingFunctions.Default(); - - Map mapOne = ImmutableMap.of(COLUMN_VISIBILITY, "A|B"); - Map mapTwo = ImmutableMap.of(COLUMN_VISIBILITY, "A|C"); + MarkingFunctions markingFunctions = new MarkingFunctions.Default(); - Map expected = ImmutableMap.of(COLUMN_VISIBILITY, "(A|B)&(A|C)"); + AccessExpressionMarkings cv1 = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression("A|B")).build(); + AccessExpressionMarkings cv2 = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression("A|C")).build(); + AccessExpressionMarkings expected = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression("(A|B)&(A|C)")).build(); - assertEquals(expected, markingFunctions.combine(mapOne, mapTwo)); + assertEquals(expected, markingFunctions.combine(cv1, cv2)); } } From a97921fe6110cafadb5b1f5aa6cbc8b528a9ea35 Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Tue, 17 Mar 2026 18:06:26 -0400 Subject: [PATCH 02/63] refactor in progress # Conflicts: # pom.xml --- .mvn/maven-build-cache-config.xml | 2 +- .../ColumnVisibilitySecurityMarking.java | 68 ++++-------- .../datawave/marking/SecurityMarking.java | 16 +-- .../query/result/event/HasMarkings.java | 6 +- .../cachedresults/CacheableQueryRowImpl.java | 102 +++++++++--------- .../CacheableQueryRowReader.java | 33 ++++-- .../common/util/AccessExpressionUtil.java | 80 ++++++++++++++ .../modification/MutableMetadataHandler.java | 45 +++++--- .../marking/AccessExpressionMarkings.java | 5 + .../datawave/marking/MarkingFunctions.java | 95 +++++----------- .../main/java/datawave/marking/Markings.java | 3 + .../FrequencyMetadataAggregator.java | 10 +- import-control-accumulo.xml | 1 + .../response/objects/DefaultKey.java | 26 ++--- .../webservice/response/objects/KeyBase.java | 5 +- .../response/objects/Visibility.java | 23 ++-- .../response/LookupResponseTest.java | 8 +- .../accumulo/admin/AdminService.java | 3 +- .../accumulo/lookup/LookupService.java | 10 +- .../dictionary/data/DefaultDescription.java | 40 +++++-- .../dictionary/data/DescriptionBase.java | 16 ++- .../dictionary/edge/MetadataBase.java | 16 +-- .../metadata/DefaultMetadataField.java | 10 +- .../data/DefaultDescriptionTest.java | 4 +- .../DataDictionaryControllerLogic.java | 9 +- .../metadata/DefaultMetadataFieldScanner.java | 6 +- .../metadata/MetadataDescriptionsHelper.java | 17 ++- .../DataDictionaryControllerTest.java | 6 +- .../data/DataDictionaryImplTest.java | 12 +-- .../DefaultMetadataFieldScannerTest.java | 2 +- .../executor/config/ExecutorProperties.java | 6 +- .../querymetric/BaseQueryMetric.java | 21 ++-- .../querymetric/QueryMetricTest.java | 17 ++- .../querymetric/QueryMetricOperations.java | 2 +- .../handler/ShardTableQueryMetricHandler.java | 2 +- .../NonWebApplicationMessagingTest.java | 4 +- .../querymetric/QueryMetricTestBase.java | 4 +- .../query/QueryManagementService.java | 6 +- .../microservice/audit/AuditClient.java | 2 +- .../CachedResultsQueryService.java | 6 +- .../config/marking/MarkingConfig.java | 2 +- pom.xml | 16 +-- .../ingest/config/RawRecordContainerImpl.java | 50 +++++---- .../ingest/data/RawRecordContainer.java | 6 +- .../data/config/BaseNormalizedContent.java | 11 +- .../ingest/data/config/MarkingsHelper.java | 26 ++--- .../config/NormalizedContentInterface.java | 7 +- .../data/config/NormalizedFieldAndValue.java | 4 +- .../data/config/ingest/CompositeIngest.java | 12 ++- .../data/config/ingest/VirtualIngest.java | 12 ++- .../handler/atom/AtomDataTypeHandler.java | 18 ++-- .../atom/AtomErrorDataTypeHandler.java | 31 +++--- .../dateindex/DateIndexDataTypeHandler.java | 5 +- .../edge/ProtobufEdgeDataTypeHandler.java | 15 ++- .../handler/edge/define/DurationValue.java | 7 +- .../handler/edge/define/EdgeDataBundle.java | 10 +- .../handler/edge/define/VertexValue.java | 7 +- .../handler/error/ErrorDataTypeHandler.java | 18 ++-- .../error/ErrorShardedDataTypeHandler.java | 7 +- .../mapreduce/handler/facet/FacetHandler.java | 5 +- .../handler/shard/ShardedDataTypeHandler.java | 19 ++-- .../data/config/MarkingsHelperTest.java | 11 +- .../config/NormalizedFieldAndValueTest.java | 15 ++- .../data/config/ingest/FakeIngestHelper.java | 7 +- .../ingest/mapreduce/SimpleRawRecord.java | 39 +++++-- .../ingest/csv/mr/input/CSVRecordReader.java | 27 ++--- .../csv/NormalizedContentInterfaceTest.java | 10 +- .../handler/ssdeep/SSDeepIndexHandler.java | 5 - ...eldIndexCountingIteratorPerVisibility.java | 4 +- .../GlobalIndexDateSummaryIterator.java | 4 +- .../iterators/ResultCountingIterator.java | 4 +- .../datawave/query/attributes/Attribute.java | 11 ++ .../query/attributes/AttributeBag.java | 13 ++- .../datawave/query/attributes/Document.java | 8 +- .../common/grouping/AbstractAggregator.java | 4 +- .../query/common/grouping/Aggregator.java | 8 +- .../common/grouping/AverageAggregator.java | 26 ++--- .../common/grouping/CountAggregator.java | 26 ++--- .../common/grouping/DocumentGrouper.java | 6 +- .../datawave/query/common/grouping/Group.java | 54 +++++----- .../query/common/grouping/GroupingUtils.java | 36 ++++--- .../query/common/grouping/MaxAggregator.java | 10 +- .../query/common/grouping/MinAggregator.java | 10 +- .../query/common/grouping/SumAggregator.java | 26 ++--- .../query/discovery/DiscoveryIterator.java | 7 +- .../query/discovery/DiscoveryTransformer.java | 18 ++-- .../query/discovery/TermInfoAggregation.java | 10 +- .../query/iterator/GroupingIterator.java | 10 +- .../iterators/FieldIndexCountingIterator.java | 4 +- .../query/jexl/DatawaveInterpreter.java | 4 +- .../jexl/functions/IdentityAggregator.java | 3 +- .../query/predicate/ValueToAttributes.java | 9 +- .../table/parser/EventKeyValueFactory.java | 21 ++-- .../parser/TermFrequencyKeyValueFactory.java | 3 +- .../shard/CountAggregatingIterator.java | 14 ++- .../shard/CountResultPostprocessor.java | 6 +- .../shard/FieldIndexCountQueryLogic.java | 4 +- .../query/tld/TLDFieldIndexAggregator.java | 4 +- .../transformer/DocumentTransformer.java | 5 +- .../DocumentTransformerSupport.java | 17 ++- .../transformer/EdgeQueryTransformer.java | 5 +- .../EdgeQueryTransformerSupport.java | 15 +-- .../transformer/EventQueryDataDecorator.java | 5 +- .../transformer/EventQueryTransformer.java | 31 +++--- .../EventQueryTransformerSupport.java | 12 ++- .../query/transformer/FacetedTransformer.java | 20 ++-- .../FieldIndexCountQueryTransformer.java | 12 ++- .../GroupingDocumentTransformer.java | 9 +- .../ShardIndexQueryTransformer.java | 12 ++- .../ShardQueryCountTableTransformer.java | 14 +-- .../transformer/TagCloudTransformer.java | 27 +++-- .../query/result/event/SimpleEvent.java | 16 +-- .../query/result/event/SimpleField.java | 35 +++--- .../common/grouping/DocumentGrouperTest.java | 40 ++++--- .../query/index/day/TruncatedIndexIngest.java | 3 +- .../tables/RemoteEventQueryLogicTest.java | 5 +- .../tables/RemoteQueryServiceTestUtil.java | 3 +- .../tables/content/ContentQueryLogicTest.java | 4 +- .../tables/keyword/TagCloudTestUtil.java | 11 +- .../ssdeep/testframework/SSDeepDataType.java | 3 +- .../query/testframework/CitiesDataType.java | 3 +- .../testframework/cardata/CarsDataType.java | 3 +- .../transformer/DocumentTransformerTest.java | 2 +- .../query/transformer/GroupingTest.java | 13 ++- .../datawave/query/util/NoUidIndexIngest.java | 3 +- .../test/java/datawave/test/GroupAssert.java | 20 ++-- .../results/cached/CachedResultsBean.java | 10 +- .../cachedresults/CacheableQueryRow.java | 15 +-- .../query/result/EdgeQueryResponseBase.java | 4 +- .../query/result/edge/DefaultEdge.java | 10 +- .../query/result/event/DefaultEvent.java | 37 +------ .../query/result/event/DefaultFacets.java | 8 +- .../query/result/event/DefaultField.java | 35 ++---- .../result/event/DefaultFieldCardinality.java | 7 +- .../query/result/event/EventBase.java | 16 +-- .../query/result/event/FacetsBase.java | 17 +-- .../query/result/event/FieldBase.java | 15 ++- .../result/event/FieldCardinalityBase.java | 14 ++- .../result/DefaultEdgeQueryResponse.java | 9 +- .../webservice/result/FacetQueryResponse.java | 8 +- .../result/FacetQueryResponseBase.java | 6 +- .../result/keyword/DefaultTagCloud.java | 34 +----- .../result/keyword/TagCloudBase.java | 17 ++- .../metrics/ShardTableQueryMetricHandler.java | 6 +- .../webservice/query/factory/Persister.java | 2 +- .../query/runner/QueryExecutorBean.java | 2 +- .../composite/CompositeQueryLogicTest.java | 7 +- 147 files changed, 1130 insertions(+), 1015 deletions(-) create mode 100644 core/common-util/src/main/java/datawave/core/common/util/AccessExpressionUtil.java diff --git a/.mvn/maven-build-cache-config.xml b/.mvn/maven-build-cache-config.xml index 1e22cfc0381..420d20c83df 100644 --- a/.mvn/maven-build-cache-config.xml +++ b/.mvn/maven-build-cache-config.xml @@ -23,7 +23,7 @@ --> - true + false SHA-256 true diff --git a/core/base-rest-responses/src/main/java/datawave/marking/ColumnVisibilitySecurityMarking.java b/core/base-rest-responses/src/main/java/datawave/marking/ColumnVisibilitySecurityMarking.java index 206dd12afd2..c2f456918a5 100644 --- a/core/base-rest-responses/src/main/java/datawave/marking/ColumnVisibilitySecurityMarking.java +++ b/core/base-rest-responses/src/main/java/datawave/marking/ColumnVisibilitySecurityMarking.java @@ -1,8 +1,5 @@ package datawave.marking; -import static com.google.common.base.Charsets.UTF_8; - -import java.util.Collections; import java.util.List; import java.util.Map; @@ -12,7 +9,8 @@ import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlAttribute; -import org.apache.accumulo.core.security.ColumnVisibility; +import org.apache.accumulo.access.Access; +import org.apache.accumulo.access.AccessExpression; import com.google.common.base.Preconditions; @@ -22,6 +20,8 @@ public class ColumnVisibilitySecurityMarking implements SecurityMarking { public static final String VISIBILITY_MARKING = "columnVisibility"; + private static final Access ACCESS = Access.builder().build(); + @XmlAttribute(name = "columnVisibility") private String columnVisibility = null; @@ -31,7 +31,7 @@ public void validate(Map> parameters) throws IllegalArgument if (null == values) { throw new IllegalArgumentException("Required parameter " + VISIBILITY_MARKING + " not found"); } - if (values.isEmpty() || values.size() > 1) { + if (values.size() != 1) { throw new IllegalArgumentException("Required parameter " + VISIBILITY_MARKING + " only accepts one value"); } columnVisibility = values.get(0); @@ -47,8 +47,16 @@ public void setColumnVisibility(String columnVisibility) { } @Override - public ColumnVisibility toColumnVisibility() { - return new ColumnVisibility(columnVisibility); + public AccessExpression toAccessExpression() { + if (columnVisibility == null || columnVisibility.isEmpty()) { + return ACCESS.newExpression(""); + } + return ACCESS.newExpression(columnVisibility); + } + + @Override + public Markings toMarkings() { + return AccessExpressionMarkings.builder().accessExpression(toAccessExpression()).build(); } @Override @@ -72,65 +80,29 @@ public boolean equals(Object obj) { if (this.columnVisibility == null && other.columnVisibility != null) { return false; } - if (this.columnVisibility == null && other.columnVisibility == null) { + if (this.columnVisibility == null) { return true; } - if (this.columnVisibility.equals(other.columnVisibility)) { - return true; - } else { - return false; - } + return this.columnVisibility.equals(other.columnVisibility); } return false; } @Override public String toString() { - return toColumnVisibilityString(); + return toAccessExpressionString(); } @Override - public String toColumnVisibilityString() { + public String toAccessExpressionString() { if (null == this.columnVisibility) { return null; } - return new String(toColumnVisibility().flatten(), UTF_8); + return toAccessExpression().getExpression(); } public void clear() { this.columnVisibility = null; } - @Override - public Map toMap() { - return Collections.singletonMap(VISIBILITY_MARKING, this.columnVisibility); - } - - @Override - public void fromMap(Map map) { - this.columnVisibility = map.get(VISIBILITY_MARKING); - } - - /** - * Turn this set of markings into a serializable string - * - * @return String - */ - @Override - public String mapToString() { - return MarkingFunctions.Encoding.toString(toMap()); - } - - /** - * Fill this security markings given an encoded string - * - * @param encodedMarkings - */ - @Override - public void fromString(String encodedMarkings) { - Map markings = MarkingFunctions.Encoding.fromString(encodedMarkings); - this.columnVisibility = markings.get(VISIBILITY_MARKING); - Preconditions.checkNotNull(columnVisibility); - } - } diff --git a/core/base-rest-responses/src/main/java/datawave/marking/SecurityMarking.java b/core/base-rest-responses/src/main/java/datawave/marking/SecurityMarking.java index d62ebd4fd1e..78b77c764b3 100644 --- a/core/base-rest-responses/src/main/java/datawave/marking/SecurityMarking.java +++ b/core/base-rest-responses/src/main/java/datawave/marking/SecurityMarking.java @@ -1,24 +1,16 @@ package datawave.marking; -import java.util.Map; - -import org.apache.accumulo.core.security.ColumnVisibility; +import org.apache.accumulo.access.AccessExpression; import datawave.validation.ParameterValidator; public interface SecurityMarking extends ParameterValidator { - ColumnVisibility toColumnVisibility() throws MarkingFunctions.Exception; - - String toColumnVisibilityString(); - - Map toMap(); - - void fromMap(Map map); + AccessExpression toAccessExpression() throws MarkingFunctions.Exception; - String mapToString(); + String toAccessExpressionString(); - void fromString(String xmlString); + Markings toMarkings() throws MarkingFunctions.Exception; void clear(); } diff --git a/core/base-rest-responses/src/main/java/datawave/webservice/query/result/event/HasMarkings.java b/core/base-rest-responses/src/main/java/datawave/webservice/query/result/event/HasMarkings.java index efc95de29a6..bfec4bcb1a7 100644 --- a/core/base-rest-responses/src/main/java/datawave/webservice/query/result/event/HasMarkings.java +++ b/core/base-rest-responses/src/main/java/datawave/webservice/query/result/event/HasMarkings.java @@ -1,10 +1,10 @@ package datawave.webservice.query.result.event; -import java.util.Map; +import datawave.marking.Markings; public interface HasMarkings { - void setMarkings(Map markings); + void setMarkings(Markings markings); - Map getMarkings(); + Markings getMarkings(); } diff --git a/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java b/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java index d50078d9b73..6e04edd6e4d 100644 --- a/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java +++ b/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java @@ -7,26 +7,29 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.TreeMap; import java.util.TreeSet; -import org.apache.accumulo.core.security.ColumnVisibility; +import org.apache.accumulo.access.AccessExpression; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.collect.Maps; import com.google.common.collect.Sets; import datawave.data.type.Type; +import datawave.marking.AccessExpressionMarkings; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.webservice.query.cachedresults.CacheableQueryRow; import datawave.webservice.query.data.ObjectSizeOf; import datawave.webservice.query.util.TypedValue; +@SuppressWarnings("unchecked") public class CacheableQueryRowImpl extends CacheableQueryRow implements ObjectSizeOf { private static Logger log = LoggerFactory.getLogger(CacheableQueryRowImpl.class); + private static final Markings EMPTY_MARKINGS = AccessExpressionMarkings.builder().accessExpression(MarkingFunctions.Default.ACCESS.newExpression("")) + .build(); private String user = null; private String queryId = null; @@ -35,31 +38,31 @@ public class CacheableQueryRowImpl extends CacheableQueryRow implements ObjectSi private String eventId = null; private String row = null; private String colFam = null; - private Map markings = new HashMap<>(); - private Map> columnMarkingsMap = new HashMap<>(); + private Markings markings = EMPTY_MARKINGS; + private Map> columnMarkingsMap = new HashMap<>(); private Map columnColumnVisibilityMap = new HashMap<>(); - private Map columnTypeMap = Maps.newHashMap(); + private Map columnTypeMap = new HashMap<>(); private Map columnTimestampMap = new HashMap<>(); private Map> columnValues = new HashMap<>(); private Set variableColumnNames = new TreeSet<>(); private String queryOrigin = null; private String resultOrigin = null; - public void addColumn(String columnName, String columnValueString, Map markings, String columnVisibility, Long timestamp) { + public void addColumn(String columnName, String columnValueString, Markings markings, String columnVisibility, Long timestamp) { addColumn(columnName, new TypedValue(columnValueString), markings, columnVisibility, timestamp); } - public void addColumn(String columnName, TypedValue columnTypedValue, Map markings, String columnVisibility, Long timestamp) { + public void addColumn(String columnName, TypedValue columnTypedValue, Markings markings, String columnVisibility, Long timestamp) { columnName = columnName.replaceAll(" ", "_"); // if new markings are the same as the old markings, skip all of this // they are the same and the markings value has already been validated - if (this.markings.equals(markings) == false) { + if (!this.markings.equals(markings)) { if (this.markings.isEmpty()) { // validate the markings try { - markingFunctions.translateToColumnVisibility(markings); + markingFunctions.toAccessExpression(markings); if (this.markings.isEmpty()) { // markings were empty, so use the one passed in. this.markings = markings; @@ -70,13 +73,13 @@ public void addColumn(String columnName, TypedValue columnTypedValue, Map columnVisibilities = Sets.newHashSet(); - columnVisibilities.add(markingFunctions.translateToColumnVisibility(this.markings)); - columnVisibilities.add(markingFunctions.translateToColumnVisibility(markings)); - ColumnVisibility combinedVisibility = markingFunctions.combine(columnVisibilities); + Set expressions = new HashSet<>(); + expressions.add(markingFunctions.toAccessExpression(this.markings)); + expressions.add(markingFunctions.toAccessExpression(markings)); + AccessExpression combined = markingFunctions.combine(expressions); // use combined marking as new markings - this.markings = markingFunctions.translateFromColumnVisibility(combinedVisibility); + this.markings = markingFunctions.fromAccessExpression(combined); } catch (MarkingFunctions.Exception e) { log.error("Invalid markings {} skipping column {} = {}", markings, columnName, columnTypedValue, e); return; @@ -98,14 +101,13 @@ public void addColumn(String columnName, TypedValue columnTypedValue, Map datawaveType, String columnName, TypedValue columnTypedValue, Map markings, - String columnVisibility) { - if (this.columnValues.containsKey(columnName) == false) { + private void manageColumnInsert(Type datawaveType, String columnName, TypedValue columnTypedValue, Markings markings, String columnVisibility) { + if (!this.columnValues.containsKey(columnName)) { Set valuesSet = Sets.newLinkedHashSet(); if (datawaveType != null) { @@ -133,18 +135,18 @@ private void manageColumnInsert(Type datawaveType, String columnName, TypedVa } else { columnValues.get(columnName).add(columnTypedValue.getValue().toString()); } - Map currMarkings = columnMarkingsMap.get(columnName); + Markings currMarkings = columnMarkingsMap.get(columnName); - if (currMarkings.equals(markings) == false) { + if (!currMarkings.equals(markings)) { try { - Set columnVisibilities = Sets.newHashSet(); - columnVisibilities.add(markingFunctions.translateToColumnVisibility(currMarkings)); - columnVisibilities.add(markingFunctions.translateToColumnVisibility(markings)); - ColumnVisibility combinedVisibility = markingFunctions.combine(columnVisibilities); - Map minMarkings = markingFunctions.translateFromColumnVisibility(combinedVisibility); + Set expressions = new HashSet<>(); + expressions.add(markingFunctions.toAccessExpression(currMarkings)); + expressions.add(markingFunctions.toAccessExpression(markings)); + AccessExpression combined = markingFunctions.combine(expressions); + Markings combinedMarkings = markingFunctions.fromAccessExpression(combined); // use combined marking as new markings - columnMarkingsMap.put(columnName, minMarkings); + columnMarkingsMap.put(columnName, combinedMarkings); // new combined marking means that this colVis is greater than old colVis columnColumnVisibilityMap.put(columnName, columnVisibility); } catch (MarkingFunctions.Exception e) { @@ -183,13 +185,13 @@ public String getColFam() { return colFam; } - public Map getMarkings() { - return Maps.newHashMap(markings); + public Markings getMarkings() { + return markings; } - public Map getColumnMarkings(String columnName) { + public Markings getColumnMarkings(String columnName) { columnName = columnName.replaceAll(" ", "_"); - Map markings = null; + Markings markings = null; if (this.columnMarkingsMap.containsKey(columnName)) { markings = this.columnMarkingsMap.get(columnName); } else { @@ -197,7 +199,7 @@ public Map getColumnMarkings(String columnName) { } if (markings == null) - markings = Maps.newHashMap(); + markings = EMPTY_MARKINGS; return markings; } @@ -209,11 +211,7 @@ public String getColumnTimestampString(Map columnMap) { Integer currColumnNumber = columnMap.get(currColumnName); if (currColumnNumber != null) { Long currLong = entry.getValue(); - Set columnSet = timestampMap.get(currLong); - if (columnSet == null) { - columnSet = new TreeSet<>(); - timestampMap.put(currLong, columnSet); - } + Set columnSet = timestampMap.computeIfAbsent(currLong, k -> new TreeSet<>()); columnSet.add(currColumnName); } } @@ -289,17 +287,17 @@ public void setColFam(String colFam) { this.colFam = colFam; } - public void setMarkings(Map markings) { + public void setMarkings(Markings markings) { // validate the markings try { - markingFunctions.translateToColumnVisibility(markings); + markingFunctions.toAccessExpression(markings); this.markings = markings; } catch (MarkingFunctions.Exception e) { log.error("Invalid markings {}", markings, e); } } - public void setColumnMarkingsMap(Map> columnMarkingsMap) { + public void setColumnMarkingsMap(Map> columnMarkingsMap) { this.columnMarkingsMap = columnMarkingsMap; } @@ -317,14 +315,16 @@ public String getColumnSecurityMarkingString(Map columnMap) { for (String field : columnMarkingsMap.keySet()) { - // sort the map - Map m = columnMarkingsMap.get(field); + Markings m = columnMarkingsMap.get(field); String v = columnColumnVisibilityMap.get(field); - if (m == null) { - m = new HashMap<>(); + String mStr = ""; + if (m != null && !m.isEmpty()) { + try { + mStr = markingFunctions.toAccessExpression(m).getExpression(); + } catch (MarkingFunctions.Exception e) { + log.error("could not serialize markings {}", m, e); + } } - // use TreeMap to sort the map - String mStr = MarkingFunctions.Encoding.toString(new TreeMap<>(m)); if (v == null) { combinedMap.put(field, mStr); } else { @@ -338,11 +338,7 @@ public String getColumnSecurityMarkingString(Map columnMap) { Map> markingToField = new HashMap<>(); for (Map.Entry entry : combinedMap.entrySet()) { String combinedMarking = entry.getValue(); - Set fields = markingToField.get(combinedMarking); - if (fields == null) { - fields = new HashSet<>(); - markingToField.put(combinedMarking, fields); - } + Set fields = markingToField.computeIfAbsent(combinedMarking, k -> new HashSet<>()); fields.add(entry.getKey()); if (fields.size() > largestSetCount) { largestSetCount = fields.size(); @@ -377,9 +373,7 @@ public void setColumnValues(Map> columnValues) { } public List getVariableColumnNames() { - List l = new ArrayList<>(); - l.addAll(variableColumnNames); - return l; + return new ArrayList<>(variableColumnNames); } public void setVariableColumnNames(Collection c) { diff --git a/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowReader.java b/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowReader.java index 94cd75e59a8..a8a2eabd465 100644 --- a/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowReader.java +++ b/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowReader.java @@ -9,10 +9,12 @@ import java.util.Set; import java.util.TreeSet; +import org.apache.accumulo.access.AccessExpression; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.webservice.query.cachedresults.CacheableQueryRow; import datawave.webservice.query.result.event.ResponseObjectFactory; @@ -20,6 +22,7 @@ public class CacheableQueryRowReader { private static Logger log = LoggerFactory.getLogger(CacheableQueryRowReader.class); + @SuppressWarnings("unchecked") public static CacheableQueryRow createRow(ResultSet cachedRowSet, Set fixedFieldsInEvent, ResponseObjectFactory responseObjectFactory, MarkingFunctions markingFunctions) { @@ -86,23 +89,37 @@ public static CacheableQueryRow createRow(ResultSet cachedRowSet, Set fi } if (columnToIndexMap.get("_markings_") != null) { String mStr = cachedRowSet.getString(columnToIndexMap.get("_markings_")); - cqfc.setMarkings(MarkingFunctions.Encoding.fromString(mStr)); + if (mStr != null && !mStr.isEmpty()) { + try { + AccessExpression ae = MarkingFunctions.Default.ACCESS.newExpression(mStr); + cqfc.setMarkings(markingFunctions.fromAccessExpression(ae)); + } catch (MarkingFunctions.Exception e) { + log.error("could not deserialize markings {}", mStr, e); + } + } } if (columnToIndexMap.get("_column_markings_") != null) { String columnMarkings = cachedRowSet.getString(columnToIndexMap.get("_column_markings_")); Map combinedColumnMarkings = parseColumnMarkings(columnMarkings, columnToIndexMap); - Map> columnMarkingsMap = new HashMap<>(); + Map> columnMarkingsMap = new HashMap<>(); Map columnVisibilityMap = new HashMap<>(); for (Map.Entry entry : combinedColumnMarkings.entrySet()) { String columnName = entry.getKey(); String combinedString = entry.getValue(); int x = combinedString.lastIndexOf(':'); - if (x >= 0) { - columnMarkingsMap.put(columnName, MarkingFunctions.Encoding.fromString(combinedString.substring(0, x))); - columnVisibilityMap.put(columnName, combinedString.substring(x + 1)); - } else { - columnMarkingsMap.put(columnName, MarkingFunctions.Encoding.fromString(combinedString)); - columnVisibilityMap.put(columnName, ""); + try { + if (x >= 0) { + String markingStr = combinedString.substring(0, x); + AccessExpression ae = MarkingFunctions.Default.ACCESS.newExpression(markingStr); + columnMarkingsMap.put(columnName, markingFunctions.fromAccessExpression(ae)); + columnVisibilityMap.put(columnName, combinedString.substring(x + 1)); + } else { + AccessExpression ae = MarkingFunctions.Default.ACCESS.newExpression(combinedString); + columnMarkingsMap.put(columnName, markingFunctions.fromAccessExpression(ae)); + columnVisibilityMap.put(columnName, ""); + } + } catch (MarkingFunctions.Exception e) { + log.error("could not deserialize column markings for {}: {}", columnName, combinedString, e); } } cqfc.setColumnMarkingsMap(columnMarkingsMap); diff --git a/core/common-util/src/main/java/datawave/core/common/util/AccessExpressionUtil.java b/core/common-util/src/main/java/datawave/core/common/util/AccessExpressionUtil.java new file mode 100644 index 00000000000..9216ccc9c30 --- /dev/null +++ b/core/common-util/src/main/java/datawave/core/common/util/AccessExpressionUtil.java @@ -0,0 +1,80 @@ +package datawave.core.common.util; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import org.apache.accumulo.access.Access; +import org.apache.accumulo.access.AccessExpression; +import org.apache.accumulo.core.security.ColumnVisibility; + +/** + * Utility for bridging between Accumulo's {@link ColumnVisibility} (from accumulo-core) and {@link AccessExpression} (from accumulo-access). + *

+ * Many modules still require {@link ColumnVisibility} for the native Accumulo Key/Mutation API, while the marking functions layer has migrated to + * {@link AccessExpression}. This utility provides zero-copy-where-possible conversions between the two. + */ +public final class AccessExpressionUtil { + + private static final Access ACCESS = Access.builder().build(); + + private static final AccessExpression EMPTY_EXPRESSION = ACCESS.newExpression(""); + + private AccessExpressionUtil() {} + + /** + * Convert a {@link ColumnVisibility} to an {@link AccessExpression}. + * + * @param cv + * the column visibility + * @return the equivalent access expression + */ + public static AccessExpression toAccessExpression(ColumnVisibility cv) { + byte[] expr = cv.getExpression(); + if (expr.length == 0) { + return EMPTY_EXPRESSION; + } + return ACCESS.newExpression(new String(expr, UTF_8)); + } + + /** + * Convert an {@link AccessExpression} to a {@link ColumnVisibility}. + * + * @param ae + * the access expression + * @return the equivalent column visibility + */ + public static ColumnVisibility toColumnVisibility(AccessExpression ae) { + String expr = ae.getExpression(); + if (expr.isEmpty()) { + return new ColumnVisibility(); + } + return new ColumnVisibility(expr); + } + + /** + * Convert a raw visibility byte array (e.g. from {@code Key.getColumnVisibilityData()}) to an {@link AccessExpression}. + * + * @param visibilityBytes + * the raw visibility bytes + * @return the equivalent access expression + */ + public static AccessExpression toAccessExpression(byte[] visibilityBytes) { + if (visibilityBytes == null || visibilityBytes.length == 0) { + return EMPTY_EXPRESSION; + } + return ACCESS.newExpression(new String(visibilityBytes, UTF_8)); + } + + /** + * @return the shared {@link Access} instance used by this utility + */ + public static Access getAccess() { + return ACCESS; + } + + /** + * @return a cached empty {@link AccessExpression} + */ + public static AccessExpression emptyExpression() { + return EMPTY_EXPRESSION; + } +} diff --git a/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java b/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java index c4af31664a1..397dee15488 100644 --- a/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java +++ b/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java @@ -45,12 +45,14 @@ import com.google.common.collect.Multimap; import datawave.core.common.connection.AccumuloConnectionFactory; +import datawave.core.common.util.AccessExpressionUtil; import datawave.core.iterators.FieldIndexDocumentFilter; import datawave.data.ColumnFamilyConstants; import datawave.data.type.Type; import datawave.ingest.protobuf.Uid; import datawave.ingest.protobuf.Uid.List.Builder; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.microservice.query.DefaultQueryParameters; import datawave.microservice.query.QueryPersistence; import datawave.modification.configuration.ModificationServiceConfiguration; @@ -310,6 +312,20 @@ public void process(AccumuloClient client, ModificationRequestBase request, Map< this.process(client, request, mutableFieldList, userAuths, userDetails, false, true); } + /** + * Convert legacy Map<String,String> field markings (from DefaultModificationRequest) to Markings<?>. + */ + private Markings convertFieldMarkings(Map rawMarkings) throws MarkingFunctions.Exception { + if (rawMarkings == null || rawMarkings.isEmpty()) { + return null; + } + String vis = rawMarkings.getOrDefault("columnVisibility", ""); + if (vis.isEmpty()) { + return null; + } + return markingFunctions.fromAccessExpression(MarkingFunctions.Default.ACCESS.newExpression(vis)); + } + public void process(AccumuloClient client, ModificationRequestBase request, Map> mutableFieldList, Set userAuths, ProxiedUserDetails userDetails, boolean purgeIndex, boolean insertHistory) throws Exception { @@ -334,7 +350,7 @@ public void process(AccumuloClient client, ModificationRequestBase request, Map< String eventUid = e.getEventUid(); String oldFieldValue = null; - Map oldFieldMarkings = null; + Markings oldFieldMarkings = null; String oldColumnVisibility = null; List> currentEntryList = null; int valHistoryCount = 0; @@ -358,8 +374,8 @@ public void process(AccumuloClient client, ModificationRequestBase request, Map< // Count the history entries if history is going to be inserted. if (insertHistory && (MODE.INSERT.equals(mode) || MODE.UPDATE.equals(mode))) { - List> fieldHistoryList = getField(client, userAuths, shardId, datatype, eventUid, "HISTORY_" + fieldName, null, - new HashMap<>(), null); + List> fieldHistoryList = getField(client, userAuths, shardId, datatype, eventUid, "HISTORY_" + fieldName, null, null, + null); for (Pair p : fieldHistoryList) { if (p.getLeft().getColumnQualifier().find(mr.getFieldValue()) > -1) { @@ -371,13 +387,13 @@ public void process(AccumuloClient client, ModificationRequestBase request, Map< if (MODE.UPDATE.equals(mode) || MODE.DELETE.equals(mode)) { if (MODE.UPDATE.equals(mode)) { oldFieldValue = mr.getOldFieldValue(); - oldFieldMarkings = mr.getOldFieldMarkings(); + oldFieldMarkings = convertFieldMarkings(mr.getOldFieldMarkings()); oldColumnVisibility = mr.getOldColumnVisibility(); if (null == oldFieldValue) throw new IllegalArgumentException("fieldValue parameter required for update"); } else { oldFieldValue = mr.getFieldValue(); - oldFieldMarkings = mr.getFieldMarkings(); + oldFieldMarkings = convertFieldMarkings(mr.getFieldMarkings()); oldColumnVisibility = mr.getColumnVisibility(); if (null == oldFieldValue) throw new IllegalArgumentException("fieldValue parameter required for delete"); @@ -399,7 +415,7 @@ public void process(AccumuloClient client, ModificationRequestBase request, Map< if (MODE.INSERT.equals(mode)) { String fieldValue = mr.getFieldValue(); - Map fieldMarkings = mr.getFieldMarkings(); + Markings fieldMarkings = convertFieldMarkings(mr.getFieldMarkings()); String columnVisibility = mr.getColumnVisibility(); ColumnVisibility colviz = null; if (null != columnVisibility) { @@ -414,7 +430,7 @@ public void process(AccumuloClient client, ModificationRequestBase request, Map< delete(writer, client, userAuths, currentEntryList, isIndexOnly, isIndexed, isReverseIndexed, isContent, dataTypes, user, MODE.UPDATE, origTimestamp + valHistoryCount, purgeIndex, insertHistory); String fieldValue = mr.getFieldValue(); - Map fieldMarkings = mr.getFieldMarkings(); + Markings fieldMarkings = convertFieldMarkings(mr.getFieldMarkings()); String columnVisibility = mr.getColumnVisibility(); ColumnVisibility colviz = null; if (null != columnVisibility) { @@ -636,7 +652,7 @@ protected void insertHistory(MultiTableBatchWriter writer, String shardId, Strin * @throws Exception * if there are issues */ - protected void insert(MultiTableBatchWriter writer, String shardId, String datatype, String eventUid, Map markings, ColumnVisibility viz, + protected void insert(MultiTableBatchWriter writer, String shardId, String datatype, String eventUid, Markings markings, ColumnVisibility viz, String fieldName, String fieldValue, boolean isIndexOnlyField, boolean isIndexed, boolean isReverseIndexed, Set> dataTypes, String user, MODE mode, long ts, boolean insertHistory) throws Exception { @@ -644,7 +660,7 @@ protected void insert(MultiTableBatchWriter writer, String shardId, String datat if (null == markings || markings.isEmpty()) throw new IllegalArgumentException("No security information specified. Security markings must be supplied"); - viz = markingFunctions.translateToColumnVisibility(markings); + viz = AccessExpressionUtil.toColumnVisibility(markingFunctions.toAccessExpression(markings)); } insert(writer, shardId, datatype, eventUid, viz, fieldName, fieldValue, ts, isIndexOnlyField, isIndexed, isReverseIndexed, dataTypes, false, @@ -816,7 +832,7 @@ protected void delete(MultiTableBatchWriter writer, AccumuloClient client, Set> getField(AccumuloClient client, Set userAuths, String shardId, String datatype, String eventUid, - String fieldName, String oldFieldValue, Map oldFieldMarkings, ColumnVisibility oldColumnVisibility) throws Exception { + String fieldName, String oldFieldValue, Markings oldFieldMarkings, ColumnVisibility oldColumnVisibility) throws Exception { Text family = new Text(datatype); TextUtil.textAppend(family, eventUid); @@ -847,15 +863,16 @@ protected List> getField(AccumuloClient client, Set markings = markingFunctions.translateFromColumnVisibilityForAuths(e.getKey().getColumnVisibilityParsed(), userAuths); + Markings markings = markingFunctions + .fromAccessExpression(AccessExpressionUtil.toAccessExpression(e.getKey().getColumnVisibilityParsed())); if (null != oldFieldMarkings && !oldFieldMarkings.equals(markings)) { log.trace("Skipping key that does not match with markings: {}", e.getKey()); continue; diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java index b924cf0018a..0776ba8be6e 100644 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java @@ -15,4 +15,9 @@ public class AccessExpressionMarkings implements Markings { public AccessExpression getMarkings() { return getAccessExpression(); } + + @Override + public boolean isEmpty() { + return accessExpression == null || accessExpression.getExpression().isEmpty(); + } } diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java index 4928cd7aa70..024579e86f9 100644 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java @@ -1,25 +1,17 @@ package datawave.marking; -import java.io.IOException; -import java.lang.reflect.InvocationTargetException; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; -import java.util.Map; import java.util.stream.Collectors; import org.apache.accumulo.access.Access; import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.access.ParsedAccessExpression; import org.apache.accumulo.access.examples.ParseExamples; -import org.apache.commons.beanutils.BeanUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.support.ClassPathXmlApplicationContext; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; - /** * Accumulo marks all data with a columnVisibility that declares and controls access. MarkingFunctions provide a pattern for mapping a user's preferred means of * declaring access controls with the Accumulo columnVisibility pattern. As an example, James Bond might use MarkingFunctions to translate a the @@ -29,13 +21,30 @@ public interface MarkingFunctions> { - AccessExpression combine(Collection columnVisibilities) throws MarkingFunctions.Exception; + AccessExpression combine(Collection expressions) throws MarkingFunctions.Exception; T combine(T... markings) throws MarkingFunctions.Exception; - AccessExpression translateToColumnVisibility(T markings) throws MarkingFunctions.Exception; + AccessExpression toAccessExpression(Markings markings) throws MarkingFunctions.Exception; + + T fromAccessExpression(AccessExpression expression) throws MarkingFunctions.Exception; - T translateFromColumnVisibility(AccessExpression columnVisibility) throws MarkingFunctions.Exception; + /** + * Normalize an {@link AccessExpression} by deduplicating and sorting terms. This is the replacement for the old {@code ColumnVisibility.flatten()} + * behavior. + * + * @param expression + * the expression to normalize + * @return a normalized {@link AccessExpression} + */ + static AccessExpression normalize(AccessExpression expression) { + String expr = expression.getExpression(); + if (expr.isEmpty()) { + return expression; + } + ParsedAccessExpression parsed = Default.ACCESS.newParsedExpression(expr); + return Default.ACCESS.newExpression(ParseExamples.normalize(parsed).expression); + } class Exception extends java.lang.Exception { @@ -65,7 +74,6 @@ class Default implements MarkingFunctions { @Override public AccessExpression combine(Collection expressions) { - // filter out any empty expressions and concatenate with '&' // @formatter:off ParsedAccessExpression expression = ACCESS.newParsedExpression(expressions @@ -84,84 +92,31 @@ public AccessExpression combine(Collection expressions) { @Override public final AccessExpressionMarkings combine(AccessExpressionMarkings... markings) { - // translate COLUMN_VISIBILITY values to ColumnVisibility, combine them and - // return translated back to Map return AccessExpressionMarkings.builder().accessExpression(combine(Arrays.stream(markings).map(Markings::getMarkings).collect(Collectors.toSet()))) .build(); } @Override - public AccessExpression translateToColumnVisibility(AccessExpressionMarkings markings) { - return markings.getMarkings(); + public AccessExpression toAccessExpression(Markings markings) { + return (AccessExpression) markings.getMarkings(); } @Override - public AccessExpressionMarkings translateFromColumnVisibility(AccessExpression expression) { + public AccessExpressionMarkings fromAccessExpression(AccessExpression expression) { return AccessExpressionMarkings.builder().accessExpression(expression).build(); } } - class Util { - - public static Object populate(Object obj, Map source) { - try { - BeanUtils.populate(obj, source); - } catch (IllegalAccessException | InvocationTargetException e) { - throw new RuntimeException("Error populating object: " + obj.getClass().getName(), e); - } - return obj; - } - } - - class Encoding { - static private Logger log = LoggerFactory.getLogger(Encoding.class); - - /** - * Turn a set of markings into a serializable string - * - * @param markings - * the markings map to convert to a string - * @return a serialized String version of {@code markings} - */ - public static String toString(Map markings) { - ObjectMapper objectMapper = new ObjectMapper(); - try { - return objectMapper.writeValueAsString(markings); - } catch (JsonProcessingException e) { - log.error("could not serialize " + markings); - return ""; - } - } - - /** - * Turn a serialized set of markings into a map - * - * @param encodedMarkings - * the serialized String markings to convert back to a markings Map - * @return a {@link Map} of the de-serialized markings from {@code encodedMarkings} - */ - public static Map fromString(String encodedMarkings) { - ObjectMapper objectMapper = new ObjectMapper(); - try { - // noinspection unchecked - return objectMapper.readValue(encodedMarkings, Map.class); - } catch (IOException e) { - log.error("could not deserialize " + encodedMarkings); - return Collections.emptyMap(); - } - } - } - /** * this Factory for MarkingFunctions is designed to be used on the tservers, where there is a vfs-classloader */ class Factory { public static final Logger log = LoggerFactory.getLogger(Factory.class); - private static MarkingFunctions markingFunctions; + private static MarkingFunctions markingFunctions; - public static synchronized MarkingFunctions createMarkingFunctions() { + public static synchronized MarkingFunctions createMarkingFunctions() { if (markingFunctions != null) return markingFunctions; ClassLoader thisClassLoader = Factory.class.getClassLoader(); diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/Markings.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/Markings.java index 4b89ef4a413..113f9a52cca 100644 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/Markings.java +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/Markings.java @@ -4,4 +4,7 @@ public interface Markings { T getMarkings(); + default boolean isEmpty() { + return getMarkings() == null; + } } diff --git a/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java b/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java index b5dfaca8076..468fe974d80 100644 --- a/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java +++ b/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java @@ -1,5 +1,7 @@ package datawave.iterators; +import static java.nio.charset.StandardCharsets.UTF_8; + import java.io.IOException; import java.util.Arrays; import java.util.Collection; @@ -10,7 +12,10 @@ import java.util.Map; import java.util.Set; import java.util.TreeMap; +import java.util.stream.Collectors; +import org.apache.accumulo.access.Access; +import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.data.ByteSequence; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; @@ -584,7 +589,10 @@ private Map buildCacheEntries() { private ColumnVisibility combineAllVisibilities() { Set visibilities = visibilityToDateFrequencies.keySet(); try { - return markingFunctions.combine(visibilities); + Access access = Access.builder().build(); + AccessExpression combined = markingFunctions + .combine(visibilities.stream().map(cv -> access.newExpression(new String(cv.getExpression(), UTF_8))).collect(Collectors.toSet())); + return new ColumnVisibility(combined.getExpression()); } catch (MarkingFunctions.Exception e) { log.error("Failed to combine visibilities " + visibilities); throw new IllegalArgumentException("Failed to combine visibilities " + visibilities, e); diff --git a/import-control-accumulo.xml b/import-control-accumulo.xml index dd6e8eb2ca9..d697b47a953 100644 --- a/import-control-accumulo.xml +++ b/import-control-accumulo.xml @@ -10,6 +10,7 @@ |datawave\.test.*| |gov.nsa.datawave.*)" regex="true" strategyOnMismatch="allowed"> + diff --git a/microservices/services/accumulo/api/src/main/java/datawave/webservice/response/objects/DefaultKey.java b/microservices/services/accumulo/api/src/main/java/datawave/webservice/response/objects/DefaultKey.java index 5fd21cff9c6..d4063faa0a7 100644 --- a/microservices/services/accumulo/api/src/main/java/datawave/webservice/response/objects/DefaultKey.java +++ b/microservices/services/accumulo/api/src/main/java/datawave/webservice/response/objects/DefaultKey.java @@ -2,16 +2,15 @@ import static java.nio.charset.StandardCharsets.UTF_8; -import java.util.HashMap; -import java.util.Map; - import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlElement; import org.apache.commons.codec.binary.Base64; +import datawave.marking.AccessExpressionMarkings; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.webservice.query.util.TypedValue; @XmlAccessorType(XmlAccessType.NONE) @@ -87,18 +86,16 @@ public long getTimestamp() { } @Override - public void setMarkings(Map markings) { - HashMap markingMap = new HashMap<>(); - if (markings != null) { - markingMap.putAll(markings); + public void setMarkings(Markings markings) { + this.markings = markings; + if (markings instanceof AccessExpressionMarkings) { + AccessExpressionMarkings aem = (AccessExpressionMarkings) markings; + this.columnVisibility = new TypedValue(aem.getAccessExpression() != null ? aem.getAccessExpression().getExpression() : ""); } - this.markings = markingMap; - - setColumnVisibility(markingMap.getOrDefault(MarkingFunctions.Default.COLUMN_VISIBILITY, "")); } @Override - public Map getMarkings() { + public Markings getMarkings() { return this.markings; } @@ -116,11 +113,8 @@ public void setColQual(String colQual) { public void setColumnVisibility(String colVis) { this.columnVisibility = (colVis == null) ? new TypedValue("") : new TypedValue(colVis); - if (markings == null) { - markings = new HashMap<>(); - } - markings.put(MarkingFunctions.Default.COLUMN_VISIBILITY, this.columnVisibility.getValue().toString()); - + String expr = this.columnVisibility.getValue().toString(); + this.markings = AccessExpressionMarkings.builder().accessExpression(MarkingFunctions.Default.ACCESS.newExpression(expr)).build(); } public void setTimestamp(long timestamp) { diff --git a/microservices/services/accumulo/api/src/main/java/datawave/webservice/response/objects/KeyBase.java b/microservices/services/accumulo/api/src/main/java/datawave/webservice/response/objects/KeyBase.java index 7b414c410d2..1d4b04aa977 100644 --- a/microservices/services/accumulo/api/src/main/java/datawave/webservice/response/objects/KeyBase.java +++ b/microservices/services/accumulo/api/src/main/java/datawave/webservice/response/objects/KeyBase.java @@ -1,18 +1,17 @@ package datawave.webservice.response.objects; -import java.util.Map; - import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlSeeAlso; +import datawave.marking.Markings; import datawave.webservice.query.result.event.HasMarkings; @XmlAccessorType(XmlAccessType.NONE) @XmlSeeAlso(DefaultKey.class) public abstract class KeyBase implements HasMarkings { - protected Map markings; + protected Markings markings; public abstract String getRow(); diff --git a/microservices/services/accumulo/api/src/main/java/datawave/webservice/response/objects/Visibility.java b/microservices/services/accumulo/api/src/main/java/datawave/webservice/response/objects/Visibility.java index 4fd67012dac..d93748785eb 100644 --- a/microservices/services/accumulo/api/src/main/java/datawave/webservice/response/objects/Visibility.java +++ b/microservices/services/accumulo/api/src/main/java/datawave/webservice/response/objects/Visibility.java @@ -1,17 +1,13 @@ package datawave.webservice.response.objects; -import java.util.HashMap; -import java.util.Map; - import javax.xml.bind.annotation.XmlAccessOrder; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorOrder; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlAttribute; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; +import javax.xml.bind.annotation.XmlTransient; -import datawave.webservice.xml.util.StringMapAdapter; +import datawave.marking.Markings; @XmlAccessorType(XmlAccessType.FIELD) @XmlAccessorOrder(XmlAccessOrder.ALPHABETICAL) @@ -20,9 +16,8 @@ public class Visibility { @XmlAttribute(name = "valid") private Boolean valid; - @XmlElement(name = "markings") - @XmlJavaTypeAdapter(StringMapAdapter.class) - private HashMap markings; + @XmlTransient + private Markings markings; @XmlAttribute(name = "visibility") private String visibility; @@ -43,16 +38,12 @@ public void setVisibility(String visibility) { this.visibility = visibility; } - public Map getMarkings() { + public Markings getMarkings() { return markings; } - public void setMarkings(Map markings) { - if (this.markings == null) { - this.markings = new HashMap(); - } - this.markings.clear(); - this.markings.putAll(markings); + public void setMarkings(Markings markings) { + this.markings = markings; } } diff --git a/microservices/services/accumulo/api/src/test/java/datawave/webservice/response/LookupResponseTest.java b/microservices/services/accumulo/api/src/test/java/datawave/webservice/response/LookupResponseTest.java index 6b5f6bfc993..c0b7c973218 100644 --- a/microservices/services/accumulo/api/src/test/java/datawave/webservice/response/LookupResponseTest.java +++ b/microservices/services/accumulo/api/src/test/java/datawave/webservice/response/LookupResponseTest.java @@ -6,9 +6,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -19,6 +17,8 @@ import com.fasterxml.jackson.databind.type.TypeFactory; import com.fasterxml.jackson.module.jaxb.JaxbAnnotationIntrospector; +import datawave.marking.AccessExpressionMarkings; +import datawave.marking.MarkingFunctions; import datawave.webservice.query.exception.QueryExceptionType; import datawave.webservice.response.objects.DefaultKey; import datawave.webservice.response.objects.Entry; @@ -39,8 +39,8 @@ private void responseSetup() { lookupResponse = new LookupResponse(); List entryList = new ArrayList<>(); for (int i = 0; i < 3; i++) { - Map markings = new HashMap<>(); - markings.put("columnVisibility", "A&B&C" + i); + AccessExpressionMarkings markings = AccessExpressionMarkings.builder().accessExpression(MarkingFunctions.Default.ACCESS.newExpression("A&B&C" + i)) + .build(); KeyBase responseKey = new DefaultKey(); responseKey.setRow("row" + i); responseKey.setColFam("cf" + i); diff --git a/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/admin/AdminService.java b/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/admin/AdminService.java index 748d809785b..6ae90b00860 100644 --- a/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/admin/AdminService.java +++ b/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/admin/AdminService.java @@ -42,6 +42,7 @@ import org.springframework.stereotype.Service; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.webservice.query.util.OptionallyEncodedString; import datawave.webservice.request.UpdateRequest; import datawave.webservice.request.objects.MutationEntry; @@ -569,7 +570,7 @@ public ValidateVisibilityResponse validateVisibilities(String[] visibilityArray) vis.setValid(false); visibilityList.add(vis); try { - Map markings = markingFunctions.translateFromColumnVisibilityForAuths(new ColumnVisibility(v), authorizations); + Markings markings = markingFunctions.fromAccessExpression(MarkingFunctions.Default.ACCESS.newExpression(v)); vis.setVisibility(v); vis.setValid(true); vis.setMarkings(markings); diff --git a/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/lookup/LookupService.java b/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/lookup/LookupService.java index 26b89ed9ef0..578fe540a20 100644 --- a/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/lookup/LookupService.java +++ b/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/lookup/LookupService.java @@ -18,7 +18,6 @@ import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.Authorizations; -import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.commons.codec.binary.Base64; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.builder.ToStringBuilder; @@ -36,6 +35,7 @@ import datawave.accumulo.util.security.UserAuthFunctions; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.marking.SecurityMarking; import datawave.microservice.accumulo.lookup.config.LookupAuditProperties; import datawave.microservice.accumulo.lookup.config.LookupProperties; @@ -218,12 +218,8 @@ public LookupResponse lookup(LookupRequest request, DatawaveUserDetails currentU continue; } - //@formatter:off - final Map markings = markingFunctions.translateFromColumnVisibilityForAuths( - new ColumnVisibility(k.getColumnVisibility()), - mergedAuths - ); - //@formatter:on + final Markings markings = markingFunctions + .fromAccessExpression(MarkingFunctions.Default.ACCESS.newExpression(k.getColumnVisibility().toString())); final KeyBase responseKey = responseObjectFactory.createKey(); responseKey.setRow(currRow); diff --git a/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java b/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java index ae9daab5890..91cbe9679fb 100644 --- a/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java +++ b/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java @@ -12,6 +12,11 @@ import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlTransient; +import org.apache.accumulo.access.AccessExpression; + +import datawave.marking.AccessExpressionMarkings; +import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.webservice.query.result.event.MapSchema; import io.protostuff.Input; import io.protostuff.Message; @@ -23,15 +28,17 @@ public class DefaultDescription extends DescriptionBase implements Serializable, Message { private static final long serialVersionUID = 6756537335803261872L; + private static final String CV_KEY = "columnVisibility"; + public DefaultDescription() {} public DefaultDescription(String description) { this.description = description; } - public DefaultDescription(String description, Map markings) { + public DefaultDescription(String description, Map markingsMap) { this.description = description; - this.markings = markings; + this.markingsMap = markingsMap; } public String getDescription() { @@ -43,12 +50,25 @@ public void setDescription(String description) { } @Override - public void setMarkings(Map markings) { - this.markings = markings; + public void setMarkings(Markings markings) { + if (markings == null || markings.isEmpty()) { + this.markingsMap = null; + } else { + Object m = markings.getMarkings(); + if (m instanceof AccessExpression) { + this.markingsMap = new HashMap<>(); + this.markingsMap.put(CV_KEY, ((AccessExpression) m).getExpression()); + } + } } - public Map getMarkings() { - return this.markings; + @Override + public Markings getMarkings() { + if (this.markingsMap != null && this.markingsMap.containsKey(CV_KEY)) { + String expr = this.markingsMap.get(CV_KEY); + return AccessExpressionMarkings.builder().accessExpression(MarkingFunctions.Default.ACCESS.newExpression(expr)).build(); + } + return null; } @Override @@ -111,8 +131,7 @@ public void writeTo(Output output, DefaultDescription message) throws IOExceptio if (message.description != null) { output.writeString(1, message.description, false); } - if (message.markings != null) - output.writeObject(1, message.markings, MapSchema.SCHEMA, false); + // markings field 2 no longer written via MapSchema; markingsMap is bridged via Markings } public void mergeFrom(Input input, DefaultDescription message) throws IOException { @@ -123,8 +142,9 @@ public void mergeFrom(Input input, DefaultDescription message) throws IOExceptio message.description = input.readString(); break; case 2: - message.markings = new HashMap(); - input.mergeObject(message.markings, MapSchema.SCHEMA); + // legacy markings map — consume but discard + HashMap legacyMarkings = new HashMap<>(); + input.mergeObject(legacyMarkings, MapSchema.SCHEMA); break; default: input.handleUnknownField(number, this); diff --git a/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java b/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java index b20e09c82d1..bf6b534f882 100644 --- a/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java +++ b/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java @@ -17,9 +17,23 @@ public abstract class DescriptionBase implements HasMarkings, Message { @XmlElement(name = "description") protected String description; + /** + * JAXB-serialized markings map. The {@link HasMarkings} interface methods {@link #getMarkings()}/{@link #setMarkings} operate on {@code Markings}; + * subclasses must bridge between this map and the Markings type. + */ @XmlElement(name = "markings") @XmlJavaTypeAdapter(StringMapAdapter.class) - protected Map markings; + protected Map markingsMap; + + /** Direct access to the JAXB markings map. */ + public Map getMarkingsMap() { + return markingsMap; + } + + /** Direct access to set the JAXB markings map. */ + public void setMarkingsMap(Map markingsMap) { + this.markingsMap = markingsMap; + } public abstract String getDescription(); diff --git a/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/edge/MetadataBase.java b/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/edge/MetadataBase.java index c134d79ab3f..926f8e9c779 100644 --- a/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/edge/MetadataBase.java +++ b/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/edge/MetadataBase.java @@ -1,14 +1,12 @@ package datawave.webservice.dictionary.edge; import java.util.List; -import java.util.Map; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlSeeAlso; -import com.google.common.collect.Maps; - +import datawave.marking.Markings; import datawave.webservice.query.result.event.HasMarkings; import io.protostuff.Message; @@ -16,19 +14,13 @@ @XmlSeeAlso(DefaultMetadata.class) public abstract class MetadataBase implements HasMarkings, Message { - protected transient Map markings; + protected transient Markings markings; - public Map getMarkings() { - assureMarkings(); + public Markings getMarkings() { return markings; } - protected void assureMarkings() { - if (this.markings == null) - this.markings = Maps.newHashMap(); - } - - public void setMarkings(Map markings) { + public void setMarkings(Markings markings) { this.markings = markings; } diff --git a/microservices/services/dictionary/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java b/microservices/services/dictionary/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java index 44cecce2691..5a9f51b550a 100644 --- a/microservices/services/dictionary/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java +++ b/microservices/services/dictionary/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java @@ -250,9 +250,10 @@ public void mergeFrom(Input input, DefaultMetadataField message) throws IOExcept int size = input.readInt32(); message.descriptions = new HashSet<>(size); for (int i = 0; i < size; i++) { - Map markings = new HashMap<>(); - input.mergeObject(markings, MapSchema.SCHEMA); - message.descriptions.add(new DefaultDescription(input.readString(), markings)); + // legacy markings map — consume but discard + HashMap legacyMarkings = new HashMap<>(); + input.mergeObject(legacyMarkings, MapSchema.SCHEMA); + message.descriptions.add(new DefaultDescription(input.readString())); } break; case 10: @@ -295,7 +296,8 @@ public void writeTo(Output output, DefaultMetadataField message) throws IOExcept output.writeInt32(9, message.getDescriptions().size(), false); for (DescriptionBase desc : message.getDescriptions()) { output.writeString(9, desc.getDescription(), true); - output.writeObject(9, desc.getMarkings(), MapSchema.SCHEMA, false); + // Write empty markings map for backward compat with old protostuff readers + output.writeObject(9, new HashMap(), MapSchema.SCHEMA, false); } output.writeString(10, message.lastUpdated, false); output.writeBool(11, message.tokenized, false); diff --git a/microservices/services/dictionary/api/src/test/java/datawave/webservice/dictionary/data/DefaultDescriptionTest.java b/microservices/services/dictionary/api/src/test/java/datawave/webservice/dictionary/data/DefaultDescriptionTest.java index a6298a171d9..aed32592d10 100644 --- a/microservices/services/dictionary/api/src/test/java/datawave/webservice/dictionary/data/DefaultDescriptionTest.java +++ b/microservices/services/dictionary/api/src/test/java/datawave/webservice/dictionary/data/DefaultDescriptionTest.java @@ -31,7 +31,7 @@ public void testMarshall() throws JAXBException { markings.put("columnVisibility", "PRIVATE"); DefaultDescription desc = new DefaultDescription(); - desc.setMarkings(markings); + desc.setMarkingsMap(markings); desc.setDescription("my description"); Set descs = new HashSet<>(); @@ -107,7 +107,7 @@ public void testUnmarshall() throws JAXBException { assertEquals("myType", f.getDatatype()); for (DefaultDescription d : f.getDescriptions()) { assertEquals("my description", d.getDescription()); - Map m = d.getMarkings(); + Map m = d.getMarkingsMap(); assertEquals("PRIVATE", m.get("columnVisibility")); markingCount = m.size(); descriptionCount++; diff --git a/microservices/services/dictionary/service/src/main/java/datawave/microservice/dictionary/DataDictionaryControllerLogic.java b/microservices/services/dictionary/service/src/main/java/datawave/microservice/dictionary/DataDictionaryControllerLogic.java index d19dc671532..0f378bd26d6 100644 --- a/microservices/services/dictionary/service/src/main/java/datawave/microservice/dictionary/DataDictionaryControllerLogic.java +++ b/microservices/services/dictionary/service/src/main/java/datawave/microservice/dictionary/DataDictionaryControllerLogic.java @@ -14,6 +14,9 @@ import com.google.common.collect.Maps; import com.google.common.collect.Multimap; +import datawave.marking.AccessExpressionMarkings; +import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.microservice.AccumuloConnectionService; import datawave.microservice.Connection; import datawave.microservice.authorization.user.DatawaveUserDetails; @@ -167,8 +170,7 @@ public VoidResponse setDescriptionPut(String fieldName, String datatype, String public VoidResponse setDescriptionPost(String fieldName, String datatype, String description, String modelName, String modelTable, String columnVisibility, DatawaveUserDetails currentUser) throws Exception { DESC desc = this.responseObjectFactory.getDescription(); - Map markings = Maps.newHashMap(); - markings.put("columnVisibility", columnVisibility); + Markings markings = AccessExpressionMarkings.builder().accessExpression(MarkingFunctions.Default.ACCESS.newExpression(columnVisibility)).build(); desc.setMarkings(markings); desc.setDescription(description); @@ -281,8 +283,7 @@ public FIELDS fieldNameDescription(String fieldName, String datatype, String mod */ public VoidResponse deleteDescription(String fieldName, String datatype, String modelName, String modelTable, String columnVisibility, DatawaveUserDetails currentUser) throws Exception { - Map markings = Maps.newHashMap(); - markings.put("columnVisibility", columnVisibility); + Markings markings = AccessExpressionMarkings.builder().accessExpression(MarkingFunctions.Default.ACCESS.newExpression(columnVisibility)).build(); DESC desc = this.responseObjectFactory.getDescription(); desc.setDescription(""); desc.setMarkings(markings); diff --git a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java index a4b5020af4b..ff525c6b4ee 100644 --- a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java +++ b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java @@ -39,13 +39,13 @@ public class DefaultMetadataFieldScanner { private static final String TIMESTAMP_FORMAT = "yyyyMMddHHmmss"; private static final Value EMPTY_VALUE = new Value(); - private final MarkingFunctions markingFunctions; + private final MarkingFunctions markingFunctions; private final ResponseObjectFactory responseObjectFactory; private final Map normalizationMap; private final Connection connectionConfig; private final int numThreads; - public DefaultMetadataFieldScanner(MarkingFunctions markingFunctions, + public DefaultMetadataFieldScanner(MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, Map normalizationMap, Connection connectionConfig, int numThreads) { this.markingFunctions = markingFunctions; @@ -275,7 +275,7 @@ private void setFieldNameAndAlias() { private void setDescriptions() throws MarkingFunctions.Exception { DefaultDescription description = responseObjectFactory.getDescription(); description.setDescription(currValue.toString()); - description.setMarkings(markingFunctions.translateFromColumnVisibility(currKey.getColumnVisibilityParsed())); + description.setMarkings(markingFunctions.fromAccessExpression(MarkingFunctions.Default.ACCESS.newExpression(currKey.getColumnVisibility().toString()))); currField.getDescriptions().add(description); } diff --git a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java index 20d0e5ccc8f..1478210b176 100644 --- a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java +++ b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java @@ -32,8 +32,10 @@ import com.google.common.collect.Multimaps; import com.google.common.collect.SetMultimap; +import datawave.core.common.util.AccessExpressionUtil; import datawave.data.ColumnFamilyConstants; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.microservice.dictionary.config.ResponseObjectFactory; import datawave.query.util.MetadataEntry; import datawave.security.util.ScannerHelper; @@ -126,8 +128,8 @@ public void setDescriptions(MetadataEntry entry, Set bw = accumuloClient.createBatchWriter(metadataTableName, bwConfig); Mutation m = new Mutation(entry.getFieldName()); for (DescriptionBase desc : descs) { - m.put(ColumnFamilyConstants.COLF_DESC, new Text(entry.getDatatype()), markingFunctions.translateToColumnVisibility(desc.getMarkings()), - new Value(desc.getDescription().getBytes(UTF_8))); + ColumnVisibility cv = AccessExpressionUtil.toColumnVisibility(markingFunctions.toAccessExpression(desc.getMarkings())); + m.put(ColumnFamilyConstants.COLF_DESC, new Text(entry.getDatatype()), cv, new Value(desc.getDescription().getBytes(UTF_8))); } bw.addMutation(m); } finally { @@ -160,7 +162,8 @@ public void removeDescription(MetadataEntry entry, DescriptionBase desc) BatchWriterConfig bwConfig = new BatchWriterConfig().setMaxMemory(10000L).setMaxLatency(100L, TimeUnit.MILLISECONDS).setMaxWriteThreads(1); bw = accumuloClient.createBatchWriter(metadataTableName, bwConfig); Mutation m = new Mutation(entry.getFieldName()); - m.putDelete(ColumnFamilyConstants.COLF_DESC, new Text(entry.getDatatype()), this.markingFunctions.translateToColumnVisibility(desc.getMarkings())); + ColumnVisibility cv = AccessExpressionUtil.toColumnVisibility(this.markingFunctions.toAccessExpression(desc.getMarkings())); + m.putDelete(ColumnFamilyConstants.COLF_DESC, new Text(entry.getDatatype()), cv); bw.addMutation(m); } finally { if (null != bw) { @@ -199,11 +202,7 @@ protected SetMultimap loadDescriptions() throws TableNotFoun } - private Map getMarkings(Key k) throws MarkingFunctions.Exception { - return getMarkings(k.getColumnVisibilityParsed()); - } - - private Map getMarkings(ColumnVisibility visibility) throws MarkingFunctions.Exception { - return markingFunctions.translateFromColumnVisibility(visibility); + private Markings getMarkings(Key k) throws MarkingFunctions.Exception { + return markingFunctions.fromAccessExpression(AccessExpressionUtil.toAccessExpression(k.getColumnVisibilityParsed())); } } diff --git a/microservices/services/dictionary/service/src/test/java/datawave/microservice/dictionary/DataDictionaryControllerTest.java b/microservices/services/dictionary/service/src/test/java/datawave/microservice/dictionary/DataDictionaryControllerTest.java index 1bc65a8e7cf..34f531566a8 100644 --- a/microservices/services/dictionary/service/src/test/java/datawave/microservice/dictionary/DataDictionaryControllerTest.java +++ b/microservices/services/dictionary/service/src/test/java/datawave/microservice/dictionary/DataDictionaryControllerTest.java @@ -3,7 +3,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import java.util.AbstractMap; -import java.util.HashMap; import java.util.Map; import org.apache.accumulo.core.client.AccumuloClient; @@ -32,7 +31,9 @@ import datawave.accumulo.inmemory.InMemoryAccumuloClient; import datawave.accumulo.inmemory.InMemoryInstance; +import datawave.marking.AccessExpressionMarkings; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.microservice.ControllerIT; import datawave.microservice.dictionary.config.DataDictionaryProperties; import datawave.webservice.dictionary.data.DefaultDataDictionary; @@ -129,8 +130,7 @@ public void testPostDescriptions() { .build(); // @formatter:on - HashMap markings = new HashMap<>(); - markings.put(MarkingFunctions.Default.COLUMN_VISIBILITY, "USER|ADMIN"); + Markings markings = AccessExpressionMarkings.builder().accessExpression(MarkingFunctions.Default.ACCESS.newExpression("USER|ADMIN")).build(); Multimap,DefaultDescription> descriptions = HashMultimap.create(); descriptions.put(new AbstractMap.SimpleEntry<>("fooField", "fooType"), new DefaultDescription("my foo field", markings)); descriptions.put(new AbstractMap.SimpleEntry<>("barField", "barType"), new DefaultDescription("my bar field", markings)); diff --git a/microservices/services/dictionary/service/src/test/java/datawave/microservice/dictionary/data/DataDictionaryImplTest.java b/microservices/services/dictionary/service/src/test/java/datawave/microservice/dictionary/data/DataDictionaryImplTest.java index 7bc49d136ae..1fadd63ab54 100644 --- a/microservices/services/dictionary/service/src/test/java/datawave/microservice/dictionary/data/DataDictionaryImplTest.java +++ b/microservices/services/dictionary/service/src/test/java/datawave/microservice/dictionary/data/DataDictionaryImplTest.java @@ -92,7 +92,7 @@ public void whenSettingDescription_givenSingleDefaultDescription_shouldSetDescri markings.put("columnVisibility", "PRIVATE"); DefaultDescription description = new DefaultDescription(); - description.setMarkings(markings); + description.setMarkingsMap(markings); description.setDescription("my ultra cool description"); // Ensure no alias will be found. @@ -114,7 +114,7 @@ public void whenSettingDescription_givenDefaultDictionaryField_shouldSetDescript markings.put("columnVisibility", "PRIVATE"); DefaultDescription description = new DefaultDescription(); - description.setMarkings(markings); + description.setMarkingsMap(markings); description.setDescription("my ultra cool description"); Set descriptions = Collections.singleton(description); @@ -142,7 +142,7 @@ public void whenSettingDescription_givenNoAlias_shouldSetDescriptionWithOriginal markings.put("columnVisibility", "PRIVATE"); DefaultDescription description = new DefaultDescription(); - description.setMarkings(markings); + description.setMarkingsMap(markings); description.setDescription("my ultra cool description"); Set descriptions = Collections.singleton(description); @@ -165,7 +165,7 @@ public void whenSettingDescription_givenAlias_shouldSetDescriptionWithAlias() th markings.put("columnVisibility", "PRIVATE"); DefaultDescription description = new DefaultDescription(); - description.setMarkings(markings); + description.setMarkingsMap(markings); description.setDescription("my ultra cool description"); Set descriptions = Collections.singleton(description); @@ -298,7 +298,7 @@ public void whenDeletingDescription_givenNoAlias_shouldDeleteDescriptionWithOrig markings.put("columnVisibility", "PRIVATE"); DefaultDescription description = new DefaultDescription(); - description.setMarkings(markings); + description.setMarkingsMap(markings); description.setDescription("my ultra cool description"); // Ensure no alias will be found. @@ -319,7 +319,7 @@ public void whenDeletingDescription_givenAlias_shouldDeleteDescriptionWithAlias( markings.put("columnVisibility", "PRIVATE"); DefaultDescription description = new DefaultDescription(); - description.setMarkings(markings); + description.setMarkingsMap(markings); description.setDescription("my ultra cool description"); // Ensure an alias will be found. diff --git a/microservices/services/dictionary/service/src/test/java/datawave/microservice/metadata/DefaultMetadataFieldScannerTest.java b/microservices/services/dictionary/service/src/test/java/datawave/microservice/metadata/DefaultMetadataFieldScannerTest.java index 894f896b8a8..2e6de88b384 100644 --- a/microservices/services/dictionary/service/src/test/java/datawave/microservice/metadata/DefaultMetadataFieldScannerTest.java +++ b/microservices/services/dictionary/service/src/test/java/datawave/microservice/metadata/DefaultMetadataFieldScannerTest.java @@ -279,7 +279,7 @@ private DefaultDescription createDescription(String descriptionText) { Map markings = new HashMap<>(); markings.put("columnVisibility", "PRIVATE}"); - description.setMarkings(markings); + description.setMarkingsMap(markings); return description; } diff --git a/microservices/services/query-executor/api/src/main/java/datawave/microservice/query/executor/config/ExecutorProperties.java b/microservices/services/query-executor/api/src/main/java/datawave/microservice/query/executor/config/ExecutorProperties.java index 8924975092d..d003a027485 100644 --- a/microservices/services/query-executor/api/src/main/java/datawave/microservice/query/executor/config/ExecutorProperties.java +++ b/microservices/services/query-executor/api/src/main/java/datawave/microservice/query/executor/config/ExecutorProperties.java @@ -2,11 +2,11 @@ import java.util.concurrent.TimeUnit; -import javax.validation.constraints.NotNull; -import javax.validation.constraints.PositiveOrZero; - import org.springframework.validation.annotation.Validated; +import jakarta.validation.constraints.NotNull; +import jakarta.validation.constraints.PositiveOrZero; + @Validated public class ExecutorProperties { diff --git a/microservices/services/query-metric/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java b/microservices/services/query-metric/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java index f4d2c88f678..dc78bb4d394 100644 --- a/microservices/services/query-metric/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java +++ b/microservices/services/query-metric/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java @@ -24,6 +24,8 @@ import javax.xml.bind.annotation.XmlElementWrapper; import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; +import org.apache.accumulo.access.Access; +import org.apache.accumulo.access.AccessExpression; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.builder.CompareToBuilder; import org.apache.commons.lang3.builder.EqualsBuilder; @@ -34,7 +36,8 @@ import com.fasterxml.jackson.annotation.JsonIgnore; -import datawave.marking.MarkingFunctions; +import datawave.marking.AccessExpressionMarkings; +import datawave.marking.Markings; import datawave.microservice.query.Query; import datawave.microservice.query.QueryImpl.Parameter; import datawave.webservice.query.exception.QueryException; @@ -1164,22 +1167,26 @@ public void setPredictions(Set predictions) { this.predictions = predictions; } + private static final Access ACCESS = Access.builder().build(); + @Override - public void setMarkings(Map markings) { + public void setMarkings(Markings markings) { if (markings == null || markings.isEmpty()) { this.columnVisibility = null; } else { - this.columnVisibility = markings.get(MarkingFunctions.Default.COLUMN_VISIBILITY); + Object m = markings.getMarkings(); + if (m instanceof AccessExpression) { + this.columnVisibility = ((AccessExpression) m).getExpression(); + } } } @Override - public Map getMarkings() { - Map markings = new HashMap<>(); + public Markings getMarkings() { if (this.columnVisibility != null) { - markings.put(MarkingFunctions.Default.COLUMN_VISIBILITY, this.columnVisibility); + return AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression(this.columnVisibility)).build(); } - return markings; + return null; } public Schema getSchemaInstance() { diff --git a/microservices/services/query-metric/api/src/test/java/datawave/microservice/querymetric/QueryMetricTest.java b/microservices/services/query-metric/api/src/test/java/datawave/microservice/querymetric/QueryMetricTest.java index 5015edee51b..1a7d289fd07 100644 --- a/microservices/services/query-metric/api/src/test/java/datawave/microservice/querymetric/QueryMetricTest.java +++ b/microservices/services/query-metric/api/src/test/java/datawave/microservice/querymetric/QueryMetricTest.java @@ -14,9 +14,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Date; -import java.util.HashMap; import java.util.List; -import java.util.Map; import javax.xml.bind.JAXBContext; import javax.xml.bind.Marshaller; @@ -28,6 +26,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.module.jaxb.JaxbAnnotationModule; +import datawave.marking.AccessExpressionMarkings; import datawave.marking.MarkingFunctions; import datawave.microservice.querymetric.BaseQueryMetric.Lifecycle; import datawave.microservice.querymetric.BaseQueryMetric.PageMetric; @@ -43,7 +42,6 @@ public class QueryMetricTest { private static QueryMetric queryMetric = null; - private static Map markings = null; private static List negativeSelectors = null; private static ArrayList pageTimes = null; private static List positiveSelectors = null; @@ -53,9 +51,8 @@ public class QueryMetricTest { @BeforeAll public static void setup() { queryMetric = new QueryMetric(); - markings = new HashMap<>(); - markings.put(MarkingFunctions.Default.COLUMN_VISIBILITY, "PUBLIC"); - queryMetric.setMarkings(markings); + queryMetric.setMarkings(AccessExpressionMarkings.builder() + .accessExpression(MarkingFunctions.Default.ACCESS.newExpression("PUBLIC")).build()); negativeSelectors = new ArrayList<>(); negativeSelectors.add("negativeSelector1"); positiveSelectors = new ArrayList<>(); @@ -104,7 +101,8 @@ public void testSettersGetters() { queryMetric.setHost("host"); queryMetric.setLastUpdated(d); queryMetric.setLifecycle(Lifecycle.INITIALIZED); - queryMetric.setMarkings(markings); + queryMetric.setMarkings(AccessExpressionMarkings.builder() + .accessExpression(MarkingFunctions.Default.ACCESS.newExpression("PUBLIC")).build()); queryMetric.setNegativeSelectors(negativeSelectors); queryMetric.setNumUpdates(0); queryMetric.setPageTimes(pageTimes); @@ -131,7 +129,7 @@ public void testSettersGetters() { assertEquals("host", queryMetric.getHost()); assertEquals(d, queryMetric.getLastUpdated()); assertEquals(Lifecycle.INITIALIZED, queryMetric.getLifecycle()); - assertEquals("PUBLIC", queryMetric.getMarkings().get(MarkingFunctions.Default.COLUMN_VISIBILITY)); + assertEquals("PUBLIC", queryMetric.getColumnVisibility()); assertEquals("negativeSelector1", queryMetric.getNegativeSelectors().get(0)); assertEquals(1, queryMetric.getNumPages()); assertEquals(0, queryMetric.getNumResults()); @@ -234,7 +232,8 @@ public void testVersionSerialization() throws Exception { qm.setHost("host"); qm.setLastUpdated(d); qm.setLifecycle(BaseQueryMetric.Lifecycle.INITIALIZED); - qm.setMarkings(markings); + qm.setMarkings(AccessExpressionMarkings.builder() + .accessExpression(MarkingFunctions.Default.ACCESS.newExpression("PUBLIC")).build()); qm.setNegativeSelectors(negativeSelectors); qm.setNumUpdates(0); qm.setPageTimes(pageTimes); diff --git a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java index 158e3d2e7f7..f2733f82d1b 100644 --- a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java +++ b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java @@ -478,7 +478,7 @@ private boolean isMetricsAdministrator(DatawaveUserDetails currentUser) { private boolean canViewMetric(DatawaveUserDetails currentUser, BaseQueryMetric metric) throws Exception { boolean userCanViewMetric = true; - ColumnVisibility columnVisibility = this.markingFunctions.translateToColumnVisibility(metric.getMarkings()); + ColumnVisibility columnVisibility = this.markingFunctions.toAccessExpression(metric.getMarkings()); for (DatawaveUser user : currentUser.getProxiedUsers()) { Authorizations authorizations = new Authorizations(user.getAuths().toArray(new String[0])); VisibilityEvaluator visibilityEvaluator = new VisibilityEvaluator(authorizations); diff --git a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java index 351ba82ad4b..d911af6ebe0 100644 --- a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java +++ b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java @@ -368,7 +368,7 @@ protected Multimap getEntries(ContentIndexingColumnBasedHan Map markings = updatedQueryMetric.getMarkings(); if (markings != null && !markings.isEmpty()) { try { - event.setVisibility(this.markingFunctions.translateToColumnVisibility(updatedQueryMetric.getMarkings())); + event.setVisibility(this.markingFunctions.toAccessExpression(updatedQueryMetric.getMarkings())); } catch (MarkingFunctions.Exception e) { log.error(e.getMessage(), e); event.setVisibility(this.queryMetricHandlerProperties.getDefaultMetricVisibility()); diff --git a/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/NonWebApplicationMessagingTest.java b/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/NonWebApplicationMessagingTest.java index 7d82ff5d9f5..4ee5321ae9d 100644 --- a/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/NonWebApplicationMessagingTest.java +++ b/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/NonWebApplicationMessagingTest.java @@ -23,7 +23,7 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit.jupiter.SpringExtension; -import datawave.marking.MarkingFunctions; +import datawave.marking.ColumnVisibilitySecurityMarking; import datawave.microservice.querymetric.config.QueryMetricTransportType; import datawave.microservice.querymetric.config.TimelyProperties; import datawave.microservice.querymetric.function.QueryMetricSupplier; @@ -53,7 +53,7 @@ public class NonWebApplicationMessagingTest { @BeforeEach public void setup() { this.metricMarkings = new HashMap<>(); - this.metricMarkings.put(MarkingFunctions.Default.COLUMN_VISIBILITY, "A&C"); + this.metricMarkings.put(ColumnVisibilitySecurityMarking.VISIBILITY_MARKING, "A&C"); this.storedMetricUpdates.clear(); } diff --git a/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java b/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java index a61f8dd3cf8..04f830e87f8 100644 --- a/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java +++ b/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java @@ -65,7 +65,7 @@ import datawave.core.common.connection.AccumuloConnectionFactory; import datawave.ingest.protobuf.Uid; -import datawave.marking.MarkingFunctions; +import datawave.marking.ColumnVisibilitySecurityMarking; import datawave.microservice.authorization.preauth.ProxiedEntityX509Filter; import datawave.microservice.authorization.user.DatawaveUserDetails; import datawave.microservice.querymetric.config.QueryMetricClientProperties; @@ -151,7 +151,7 @@ public class QueryMetricTestBase { static { metricMarkings = new HashMap<>(); - metricMarkings.put(MarkingFunctions.Default.COLUMN_VISIBILITY, "A&C"); + metricMarkings.put(ColumnVisibilitySecurityMarking.VISIBILITY_MARKING, "A&C"); } @AfterAll diff --git a/microservices/services/query/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/microservices/services/query/service/src/main/java/datawave/microservice/query/QueryManagementService.java index a16e1816bdd..0898c12d14f 100644 --- a/microservices/services/query/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/microservices/services/query/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -2174,7 +2174,7 @@ public void audit(String auditId, Auditor.AuditType auditType, String logicName, // if we haven't already, validate the markings SecurityMarking requestSecurityMarking = querySecurityMarking.get(); - if (requestSecurityMarking.toColumnVisibilityString() == null) { + if (requestSecurityMarking.toAccessExpressionString() == null) { validateSecurityMarkings(parameters); } @@ -2359,7 +2359,7 @@ protected Query createQuery(String queryLogicName, MultiValueMap Query query = responseObjectFactory.getQueryImpl(); query.initialize(userDn, dnList, queryLogicName, requestQueryParameters, requestQueryParameters.getUnknownParameters(parameters)); - query.setColumnVisibility(requestSecurityMarking.toColumnVisibilityString()); + query.setColumnVisibility(requestSecurityMarking.toAccessExpressionString()); query.setUncaughtExceptionHandler(new QueryUncaughtExceptionHandler()); Thread.currentThread().setUncaughtExceptionHandler(query.getUncaughtExceptionHandler()); if (queryId != null) { @@ -2602,7 +2602,7 @@ public void setInternalAuditParameters(String queryLogicName, String userDn, Mul // These are parameters that aren't passed in by the user, but rather are computed from other sources. PrivateAuditConstants.stripPrivateParameters(parameters); parameters.add(PrivateAuditConstants.LOGIC_CLASS, queryLogicName); - parameters.set(PrivateAuditConstants.COLUMN_VISIBILITY, querySecurityMarking.get().toColumnVisibilityString()); + parameters.set(PrivateAuditConstants.COLUMN_VISIBILITY, querySecurityMarking.get().toAccessExpressionString()); parameters.add(PrivateAuditConstants.USER_DN, userDn); } diff --git a/microservices/starters/audit/src/main/java/datawave/microservice/audit/AuditClient.java b/microservices/starters/audit/src/main/java/datawave/microservice/audit/AuditClient.java index b0167b38a61..4709b2b64c8 100644 --- a/microservices/starters/audit/src/main/java/datawave/microservice/audit/AuditClient.java +++ b/microservices/starters/audit/src/main/java/datawave/microservice/audit/AuditClient.java @@ -165,7 +165,7 @@ protected Request(Builder b) throws IllegalArgumentException { params.set(AuditParameters.QUERY_AUDIT_TYPE, b.auditType.name()); } if (null != b.marking) { - params.set(AuditParameters.QUERY_SECURITY_MARKING_COLVIZ, b.marking.toColumnVisibilityString()); + params.set(AuditParameters.QUERY_SECURITY_MARKING_COLVIZ, b.marking.toAccessExpressionString()); } if (null != b.queryLogic) { params.set(AuditParameters.QUERY_LOGIC_CLASS, b.queryLogic); diff --git a/microservices/starters/cached-results/src/main/java/datawave/microservice/query/cachedresults/CachedResultsQueryService.java b/microservices/starters/cached-results/src/main/java/datawave/microservice/query/cachedresults/CachedResultsQueryService.java index cb294c76221..24b90230e09 100644 --- a/microservices/starters/cached-results/src/main/java/datawave/microservice/query/cachedresults/CachedResultsQueryService.java +++ b/microservices/starters/cached-results/src/main/java/datawave/microservice/query/cachedresults/CachedResultsQueryService.java @@ -428,7 +428,7 @@ public void setValues(PreparedStatement ps, int i) throws SQLException { ps.setString(5, cacheableQueryRow.getEventId()); ps.setString(6, cacheableQueryRow.getRow()); ps.setString(7, cacheableQueryRow.getColFam()); - ps.setString(8, MarkingFunctions.Encoding.toString(new TreeMap<>(cacheableQueryRow.getMarkings()))); + ps.setString(8, cacheableQueryRow.getMarkings().toString()); // keep track of the populated columns Set populatedColumns = new HashSet<>(); @@ -769,7 +769,7 @@ public void audit(String auditId, Auditor.AuditType auditType, String logicName, // if we haven't already, validate the markings SecurityMarking securityMarking = scopedSecurityMarking.get(); - if (securityMarking.toColumnVisibilityString() == null) { + if (securityMarking.toAccessExpressionString() == null) { validateSecurityMarkings(parameters); } @@ -817,7 +817,7 @@ public void setInternalAuditParameters(String queryLogicName, String userDn, Mul // These are parameters that aren't passed in by the user, but rather are computed from other sources. PrivateAuditConstants.stripPrivateParameters(parameters); parameters.add(PrivateAuditConstants.LOGIC_CLASS, queryLogicName); - parameters.set(PrivateAuditConstants.COLUMN_VISIBILITY, scopedSecurityMarking.get().toColumnVisibilityString()); + parameters.set(PrivateAuditConstants.COLUMN_VISIBILITY, scopedSecurityMarking.get().toAccessExpressionString()); parameters.add(PrivateAuditConstants.USER_DN, userDn); } diff --git a/microservices/starters/datawave/src/main/java/datawave/microservice/config/marking/MarkingConfig.java b/microservices/starters/datawave/src/main/java/datawave/microservice/config/marking/MarkingConfig.java index 0160a938824..0bbfa80d74c 100644 --- a/microservices/starters/datawave/src/main/java/datawave/microservice/config/marking/MarkingConfig.java +++ b/microservices/starters/datawave/src/main/java/datawave/microservice/config/marking/MarkingConfig.java @@ -24,7 +24,7 @@ public class MarkingConfig { @Bean @ConditionalOnMissingBean @ConditionalOnProperty(name = "datawave.defaults.MarkingFunctions.enabled", havingValue = "true", matchIfMissing = true) - public MarkingFunctions markingFunctions() { + public MarkingFunctions markingFunctions() { return new MarkingFunctions.Default(); } diff --git a/pom.xml b/pom.xml index 44d20522570..80a0f990dfd 100644 --- a/pom.xml +++ b/pom.xml @@ -75,14 +75,14 @@ 1.11.0 5.9.0 5.9.0 - 4.0.0 - 4.0.0 - 4.0.0 - 4.0.5 - 1.0.0 - 3.0.0 - 1.0.0 - 4.0.7 + 4.0.2-SNAPSHOT + 4.0.3-SNAPSHOT + 4.0.3-SNAPSHOT + 4.0.7-SNAPSHOT + 1.0.2-SNAPSHOT + 3.0.1-SNAPSHOT + 1.0.2-SNAPSHOT + 4.1.4-SNAPSHOT 1.9.0 3.6.3 5.2.0 diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/config/RawRecordContainerImpl.java b/warehouse/ingest-core/src/main/java/datawave/ingest/config/RawRecordContainerImpl.java index 96dea3dc999..23ad57d8da6 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/config/RawRecordContainerImpl.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/config/RawRecordContainerImpl.java @@ -15,8 +15,9 @@ import java.util.Set; import java.util.concurrent.ConcurrentSkipListSet; +import org.apache.accumulo.access.Access; +import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.security.ColumnVisibility; -import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.ToStringBuilder; import org.apache.hadoop.conf.Configurable; @@ -36,7 +37,8 @@ import datawave.ingest.data.config.ConfigurationHelper; import datawave.ingest.data.config.ingest.IgnorableErrorHelperInterface; import datawave.ingest.protobuf.RawRecordContainer.Data; -import datawave.marking.MarkingFunctions; +import datawave.marking.AccessExpressionMarkings; +import datawave.marking.Markings; import datawave.util.CompositeTimestamp; public class RawRecordContainerImpl implements Writable, Configurable, RawRecordContainer { @@ -80,57 +82,61 @@ public class RawRecordContainerImpl implements Writable, Configurable, RawRecord private Object auxData = null; private Map auxMap = null; + private static final Access ACCESS = Access.builder().build(); + // RawRecordContainer support - Map securityMarkings = null; + Markings securityMarkings = null; public RawRecordContainerImpl() { uidBuilder = UID.builder(); } @Override - public Map getSecurityMarkings() { + public Markings getSecurityMarkings() { return this.securityMarkings; } @Override - public void setSecurityMarkings(Map securityMarkings) { - this.securityMarkings = (securityMarkings == null ? null : new HashMap<>(securityMarkings)); + public void setSecurityMarkings(Markings securityMarkings) { + this.securityMarkings = securityMarkings; syncSecurityMarkingsToFields(); } @Override public void addSecurityMarking(String domain, String marking) { - if (null == securityMarkings) { - securityMarkings = new HashMap<>(); - } - securityMarkings.put(domain, marking); + // Legacy map-based operation — treat marking as an access expression string + this.securityMarkings = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression(marking)).build(); syncSecurityMarkingsToFields(); } @Override public boolean hasSecurityMarking(String domain, String marking) { - return null != securityMarkings && securityMarkings.containsKey(domain) && StringUtils.equals(securityMarkings.get(domain), marking); + if (securityMarkings == null) { + return false; + } + Object m = securityMarkings.getMarkings(); + if (m instanceof AccessExpression) { + return ((AccessExpression) m).getExpression().equals(marking); + } + return false; } protected void syncSecurityMarkingsToFields() { if (securityMarkings != null) { - setVisibility(securityMarkings.get(MarkingFunctions.Default.COLUMN_VISIBILITY)); + Object m = securityMarkings.getMarkings(); + if (m instanceof AccessExpression) { + setVisibilityNoSync(((AccessExpression) m).getExpression()); + } } else { - setVisibility((String) null); + setVisibilityNoSync(null); } } protected void syncFieldsToSecurityMarkings() { if (visibility != null) { - if (securityMarkings == null) { - securityMarkings = new HashMap<>(); - } - securityMarkings.put(MarkingFunctions.Default.COLUMN_VISIBILITY, new String(visibility.getExpression())); - - } else if (securityMarkings != null) { - securityMarkings.remove(MarkingFunctions.Default.COLUMN_VISIBILITY); - } - if (securityMarkings != null && securityMarkings.isEmpty()) { + String expr = new String(visibility.getExpression()); + securityMarkings = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression(expr)).build(); + } else { securityMarkings = null; } } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/RawRecordContainer.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/RawRecordContainer.java index 756f8bb26e7..1e0106022ef 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/RawRecordContainer.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/RawRecordContainer.java @@ -4,11 +4,11 @@ import java.io.IOException; import java.util.Collection; import java.util.Date; -import java.util.Map; import org.apache.accumulo.core.security.ColumnVisibility; import datawave.data.hash.UID; +import datawave.marking.Markings; import datawave.util.CompositeTimestamp; /** @@ -17,9 +17,9 @@ */ public interface RawRecordContainer { - Map getSecurityMarkings(); + Markings getSecurityMarkings(); - void setSecurityMarkings(Map securityMarkings); + void setSecurityMarkings(Markings securityMarkings); void addSecurityMarking(String domain, String marking); diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/BaseNormalizedContent.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/BaseNormalizedContent.java index 4737e284ad4..3373d4e709d 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/BaseNormalizedContent.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/BaseNormalizedContent.java @@ -1,11 +1,10 @@ package datawave.ingest.data.config; -import java.util.Map; - import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; import datawave.data.type.Type; +import datawave.marking.Markings; /** * Base container class implementation of the NormalizedContentInterface. This class is utilized to retain the original and transformed content and labels for a @@ -22,7 +21,7 @@ public class BaseNormalizedContent implements NormalizedContentInterface, Clonea protected String _indexedFieldValue = null; /** The security markings for the field value pair. */ - protected Map _markings = null; + protected Markings _markings = null; /** The field processing error if any. */ protected Throwable error = null; @@ -40,7 +39,7 @@ public BaseNormalizedContent(String field, String value) { _indexedFieldValue = value; } - public BaseNormalizedContent(String field, String value, Map markings) { + public BaseNormalizedContent(String field, String value, Markings markings) { this(field, value); _markings = markings; } @@ -117,12 +116,12 @@ public void setEventFieldValue(String originalFieldValue) { } @Override - public Map getMarkings() { + public Markings getMarkings() { return _markings; } @Override - public void setMarkings(Map markings) { + public void setMarkings(Markings markings) { hashCode = null; this._markings = markings; } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/MarkingsHelper.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/MarkingsHelper.java index 1f61a7a1e8b..48719633519 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/MarkingsHelper.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/MarkingsHelper.java @@ -4,11 +4,13 @@ import java.util.Map; import java.util.Map.Entry; +import org.apache.accumulo.access.Access; import org.apache.hadoop.conf.Configuration; import datawave.ingest.data.Type; import datawave.ingest.data.config.ingest.BaseIngestHelper; -import datawave.marking.MarkingFunctions; +import datawave.marking.AccessExpressionMarkings; +import datawave.marking.Markings; public interface MarkingsHelper { @@ -28,7 +30,7 @@ public interface MarkingsHelper { * * @return markings */ - Map getDefaultMarkings(); + Markings getDefaultMarkings(); /** * Returns the markings override for the field @@ -37,7 +39,7 @@ public interface MarkingsHelper { * the field name * @return markings */ - Map getFieldMarking(String fieldName); + Markings getFieldMarking(String fieldName); /** * Method to set the markings on a field @@ -48,12 +50,14 @@ public interface MarkingsHelper { void markField(NormalizedContentInterface field); /** - * No-op helper for default ColumnVisibility implementation. Should only be used for testing purposes. + * No-op helper for default AccessExpression implementation. Should only be used for testing purposes. */ class NoOp implements MarkingsHelper { - private Map> fieldMarkingMap = new HashMap<>(); - private Map defaultMarkings = null; + private static final Access ACCESS = Access.builder().build(); + + private Map> fieldMarkingMap = new HashMap<>(); + private Markings defaultMarkings = null; Configuration conf; Type dataType; @@ -67,15 +71,13 @@ public NoOp(Configuration conf, Type dataType) { private void initDefaultMarkings() { String marking = conf.get(dataType.typeName() + DEFAULT_MARKING); if (null != marking) { - defaultMarkings = new HashMap<>(); - defaultMarkings.put(MarkingFunctions.Default.COLUMN_VISIBILITY, marking); + defaultMarkings = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression(marking)).build(); } for (Entry property : conf) { if (property.getKey().startsWith(dataType.typeName()) && property.getKey().endsWith(FIELD_MARKING)) { String fieldName = null; if (null != (fieldName = BaseIngestHelper.getFieldName(dataType, property.getKey(), FIELD_MARKING))) { - Map fieldMarking = new HashMap<>(); - fieldMarking.put(MarkingFunctions.Default.COLUMN_VISIBILITY, property.getValue()); + Markings fieldMarking = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression(property.getValue())).build(); fieldMarkingMap.put(fieldName, fieldMarking); } } @@ -83,12 +85,12 @@ private void initDefaultMarkings() { } @Override - public Map getDefaultMarkings() { + public Markings getDefaultMarkings() { return defaultMarkings; } @Override - public Map getFieldMarking(String fieldName) { + public Markings getFieldMarking(String fieldName) { return fieldMarkingMap.get(fieldName); } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/NormalizedContentInterface.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/NormalizedContentInterface.java index 86c5df3eef1..1061afacb99 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/NormalizedContentInterface.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/NormalizedContentInterface.java @@ -1,8 +1,7 @@ package datawave.ingest.data.config; -import java.util.Map; - import datawave.data.type.Type; +import datawave.marking.Markings; /** * Interface utilized to reference the original and transformed content and labels for a value pair ingested into accumulo. @@ -66,7 +65,7 @@ public interface NormalizedContentInterface extends Cloneable { * @param markings * the markings to set */ - void setMarkings(Map markings); + void setMarkings(Markings markings); /** * When we fail to process a field (e.g. normalize), then an error can be set which in the EventMapper results in the event being dropped from normal @@ -82,7 +81,7 @@ public interface NormalizedContentInterface extends Cloneable { * * @return The field specific markings, or null if the overall event markings are to be used. */ - Map getMarkings(); + Markings getMarkings(); /** * Get the processing error for this field if any. diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/NormalizedFieldAndValue.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/NormalizedFieldAndValue.java index f1adc065713..c7a5e67060d 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/NormalizedFieldAndValue.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/NormalizedFieldAndValue.java @@ -6,7 +6,6 @@ * will be used. */ import java.io.UnsupportedEncodingException; -import java.util.Map; import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; @@ -15,6 +14,7 @@ import datawave.ingest.config.IngestConfiguration; import datawave.ingest.config.IngestConfigurationFactory; import datawave.ingest.config.MimeDecoder; +import datawave.marking.Markings; public class NormalizedFieldAndValue extends BaseNormalizedContent implements GroupedNormalizedContentInterface { private boolean grouped = false; @@ -42,7 +42,7 @@ public NormalizedFieldAndValue(String field, String value) { super(field, value); } - public NormalizedFieldAndValue(String field, String value, Map markings) { + public NormalizedFieldAndValue(String field, String value, Markings markings) { super(field, value, markings); } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CompositeIngest.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CompositeIngest.java index a5fef654c62..d607ed7a1dd 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CompositeIngest.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CompositeIngest.java @@ -30,6 +30,7 @@ import datawave.ingest.data.config.NormalizedContentInterface; import datawave.ingest.data.config.NormalizedFieldAndValue; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; /** * Similar to VirtualIngest virtual fields, but composite fields are not written in the event section of the shard table (only in the field index section) @@ -286,15 +287,16 @@ private String[] getConfPrefixes(String type) { } /** - * A helper routine to merge markings maps when merging fields of a NormalizedContentInterface + * A helper routine to merge markings when merging fields of a NormalizedContentInterface * * @param markings1 - * a map of markings + * markings * @param markings2 - * a different map of markings + * different markings * @return the merged markings */ - protected Map mergeMarkings(Map markings1, Map markings2) { + @SuppressWarnings("unchecked") + protected Markings mergeMarkings(Markings markings1, Markings markings2) { if (markings2 != null) { if (markings1 == null) { markings1 = markings2; @@ -356,7 +358,7 @@ private void compilePatterns() { // Create the composite field from the event Fields members public void addCompositeFields(List compositeFields, Multimap eventFields, String compositeFieldName, String replacement, String[] grouping, GroupingPolicy groupingPolicy, boolean allowMissing, String[] fields, - int pos, StringBuilder originalValue, StringBuilder normalizedValue, Map markings, boolean isOverloadedField) { + int pos, StringBuilder originalValue, StringBuilder normalizedValue, Markings markings, boolean isOverloadedField) { String separator = (pos > 0) ? compositeSeparator.get(compositeFieldName) : ""; // append any constants that have been specified while (pos < fields.length && isConstant(fields[pos])) { diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java index b24bb798f9f..119eec308e1 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java @@ -28,6 +28,7 @@ import datawave.ingest.data.config.NormalizedContentInterface; import datawave.ingest.data.config.NormalizedFieldAndValue; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; public interface VirtualIngest { @@ -339,15 +340,16 @@ private String[] getConfPrefixes(String type, String instance) { } /** - * A helper routine to merge markings maps when merging fields of a NormalizedContentInterface + * A helper routine to merge markings when merging fields of a NormalizedContentInterface * * @param markings1 - * a map of markings + * markings * @param markings2 - * a different map of markings + * different markings * @return the merged markings */ - protected Map mergeMarkings(Map markings1, Map markings2) { + @SuppressWarnings("unchecked") + protected Markings mergeMarkings(Markings markings1, Markings markings2) { if (markings2 != null) { if (markings1 == null) { markings1 = markings2; @@ -544,7 +546,7 @@ public String toString() { public void addVirtualFields(List virtualFields, Multimap eventFields, Map> groupings, String virtualFieldName, String replacement, VirtualFieldGrouping grouping, GroupingPolicy groupingPolicy, boolean allowMissing, String[] fields, int pos, String startSeparator, - String endSeparator, StringBuilder originalValue, StringBuilder normalizedValue, Map markings) { + String endSeparator, StringBuilder originalValue, StringBuilder normalizedValue, Markings markings) { String separator = ""; // append any constants that have been specified while (pos < fields.length && isConstant(fields[pos])) { diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomDataTypeHandler.java index 5ba56f69fda..86ea38b88bf 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomDataTypeHandler.java @@ -19,6 +19,7 @@ import com.google.common.base.Strings; import com.google.common.collect.Multimap; +import datawave.core.common.util.AccessExpressionUtil; import datawave.ingest.data.RawRecordContainer; import datawave.ingest.data.Type; import datawave.ingest.data.TypeRegistry; @@ -30,6 +31,7 @@ import datawave.ingest.mapreduce.job.writer.ContextWriter; import datawave.ingest.metadata.RawRecordMetadata; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.util.StringUtils; import datawave.util.TextUtil; @@ -255,28 +257,28 @@ public Key createKey(String fieldName, String fieldValue, String columnQualifier * @return the visibility */ + @SuppressWarnings("unchecked") protected String getColumnQualifier(RawRecordContainer event, NormalizedContentInterface value) { - ColumnVisibility visibility = event.getVisibility(); - if (value.getMarkings() != null && !value.getMarkings().isEmpty()) { + Markings markings = value.getMarkings(); + if (markings != null && !markings.isEmpty()) { try { - visibility = markingFunctions.translateToColumnVisibility(value.getMarkings()); + return markingFunctions.toAccessExpression(markings).getExpression(); } catch (MarkingFunctions.Exception e) { throw new RuntimeException("Cannot convert record-level markings into a column visibility", e); - } } - return flatten(visibility); + return flatten(event.getVisibility()); } /** - * Create a flattened visibility, using the cache if possible + * Normalize a ColumnVisibility expression via AccessExpression. * * @param vis * the column visibility - * @return the flattened visibility + * @return the normalized visibility string */ protected String flatten(ColumnVisibility vis) { - return new String(markingFunctions.flatten(vis)); + return MarkingFunctions.normalize(AccessExpressionUtil.toAccessExpression(vis)).getExpression(); } } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomErrorDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomErrorDataTypeHandler.java index aa8c731581c..d6b26e0cd6a 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomErrorDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomErrorDataTypeHandler.java @@ -5,6 +5,7 @@ import java.util.HashSet; import java.util.Set; +import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.hadoop.conf.Configuration; @@ -29,6 +30,7 @@ import datawave.ingest.mapreduce.job.BulkIngestKey; import datawave.ingest.mapreduce.job.writer.ContextWriter; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; /** * This class differs from the parent in that when it sees a field name of ERROR it creates a category name using the field name and value. @@ -48,10 +50,8 @@ public class AtomErrorDataTypeHandler extends AtomDataTyp public static final String ERROR_FIELD = "ERROR"; public static final String STACK_TRACE_FIELD = "STACKTRACE"; public static final String EVENT_CONTENT_FIELD = "EVENT"; - public static final String COLUMN_VISIBILITY = "columnVisibility"; - private MarkingsHelper mHelper = null; - private ColumnVisibility defaultVisibility = null; + private AccessExpression defaultVisibility = null; private Configuration conf = null; private ErrorShardedIngestHelper errorHelper = null; private MarkingFunctions markingFunctions; @@ -78,8 +78,8 @@ public long process(KEYIN key, RawRecordContainer record, Multimap markings = value.getMarkings(); + if (markings != null && !markings.isEmpty()) { try { - visibility = markingFunctions.translateToColumnVisibility(value.getMarkings()); + return markingFunctions.toAccessExpression(markings).getExpression(); } catch (MarkingFunctions.Exception e) { throw new RuntimeException("Cannot convert record-level markings into a column visibility", e); - } } - return visibility == null ? defaultVisibility : visibility; + ColumnVisibility vis = event.getVisibility(); + if (vis != null) { + return new String(vis.getExpression()); + } + return defaultVisibility != null ? defaultVisibility.getExpression() : ""; } } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/dateindex/DateIndexDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/dateindex/DateIndexDataTypeHandler.java index 11a74c42307..65c507e7067 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/dateindex/DateIndexDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/dateindex/DateIndexDataTypeHandler.java @@ -1,5 +1,7 @@ package datawave.ingest.mapreduce.handler.dateindex; +import static java.nio.charset.StandardCharsets.UTF_8; + import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; @@ -26,6 +28,7 @@ import com.google.common.collect.Multimap; import datawave.core.common.logging.ThreadConfigurableLogger; +import datawave.core.common.util.AccessExpressionUtil; import datawave.data.normalizer.DateNormalizer; import datawave.ingest.data.RawRecordContainer; import datawave.ingest.data.Type; @@ -364,7 +367,7 @@ private Value createDateIndexValue(int shard) { * @return the flattened visibility */ protected byte[] flatten(ColumnVisibility vis) { - return MarkingFunctions.Factory.createMarkingFunctions().flatten(vis); + return MarkingFunctions.normalize(AccessExpressionUtil.toAccessExpression(vis)).getExpression().getBytes(UTF_8); } public Text getDateIndexTableName() { diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDataTypeHandler.java index 7c70e7235a4..ca82592aebe 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDataTypeHandler.java @@ -1,5 +1,7 @@ package datawave.ingest.mapreduce.handler.edge; +import static java.nio.charset.StandardCharsets.UTF_8; + import static datawave.ingest.mapreduce.handler.shard.ShardedDataTypeHandler.METADATA_TABLE_LOADER_PRIORITY; import static datawave.ingest.mapreduce.handler.shard.ShardedDataTypeHandler.METADATA_TABLE_NAME; @@ -20,6 +22,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.ColumnVisibility; @@ -44,6 +47,7 @@ import com.google.common.hash.Hashing; import com.google.common.hash.PrimitiveSink; +import datawave.core.common.util.AccessExpressionUtil; import datawave.data.normalizer.DateNormalizer; import datawave.edge.util.EdgeKey; import datawave.edge.util.EdgeKey.EDGE_FORMAT; @@ -74,6 +78,7 @@ import datawave.ingest.table.config.LoadDateTableConfigHelper; import datawave.ingest.time.Now; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.metadata.protobuf.EdgeMetadata; import datawave.metadata.protobuf.EdgeMetadata.MetadataValue; import datawave.metadata.protobuf.EdgeMetadata.MetadataValue.Metadata; @@ -1327,14 +1332,16 @@ public static Map> getLocalityGroups(Configuration conf) { * the event container * @return the visibility as Text object */ - protected Text getVisibility(Map markings, RawRecordContainer event) { + @SuppressWarnings("unchecked") + protected Text getVisibility(Markings markings, RawRecordContainer event) { try { if (null == markings || markings.isEmpty()) { return new Text(flatten(event.getVisibility())); } else { - return new Text(flatten(markingFunctions.translateToColumnVisibility(markings))); + AccessExpression ae = MarkingFunctions.normalize(markingFunctions.toAccessExpression(markings)); + return new Text(ae.getExpression().getBytes(UTF_8)); } - } catch (datawave.marking.MarkingFunctions.Exception e) { + } catch (MarkingFunctions.Exception e) { throw new RuntimeException("Cannot convert markings into column visibility", e); } } @@ -1363,7 +1370,7 @@ protected Text getDurationVisibility(EdgeDataBundle value) { * @return the flattened visibility */ protected byte[] flatten(ColumnVisibility vis) { - return markingFunctions.flatten(vis); + return MarkingFunctions.normalize(AccessExpressionUtil.toAccessExpression(vis)).getExpression().getBytes(UTF_8); } @Override diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/DurationValue.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/DurationValue.java index df3f9023476..33b9f093b48 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/DurationValue.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/DurationValue.java @@ -1,8 +1,7 @@ package datawave.ingest.mapreduce.handler.edge.define; -import java.util.Map; - import datawave.ingest.data.config.NormalizedContentInterface; +import datawave.marking.Markings; /** * Extracts the duration field specified by a Normalized Content Interface @@ -13,7 +12,7 @@ public class DurationValue { private int duration = -1; - private Map markings = null; + private Markings markings = null; public DurationValue(NormalizedContentInterface elapsedTimeNCI) { String durString = elapsedTimeNCI.getIndexedFieldValue(); @@ -36,7 +35,7 @@ public int getDuration() { } } - public Map getMarkings() { + public Markings getMarkings() { return this.markings; } } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/EdgeDataBundle.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/EdgeDataBundle.java index 77d25163774..d4659e72bad 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/EdgeDataBundle.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/EdgeDataBundle.java @@ -3,7 +3,6 @@ import java.io.IOException; import java.util.Calendar; import java.util.List; -import java.util.Map; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.ColumnVisibility; @@ -21,6 +20,7 @@ import datawave.ingest.mapreduce.handler.edge.define.VertexValue.ValueType; import datawave.marking.MarkingFunctions; import datawave.marking.MarkingFunctions.Exception; +import datawave.marking.Markings; import datawave.util.time.DateHelper; /** @@ -55,7 +55,7 @@ public class EdgeDataBundle { private long eventDate = 0; private IngestHelperInterface helper; - private Map markings = null; + private Markings markings = null; private ColumnVisibility maskedVisibility = null; private boolean forceMaskedVisibility = false; // if this is a masked event, but the // the fields defined aren't masked, use the unmasked visibility @@ -141,7 +141,7 @@ public void setMaskedVisibility(ColumnVisibility maskedVisibility) { } @SuppressWarnings("unchecked") - private void initMarkings(Map m1, Map m2) { + private void initMarkings(Markings m1, Markings m2) { if (m1 != null) { if (m2 != null) { try { @@ -385,11 +385,11 @@ public void setSink(VertexValue sink) { this.sink = sink; } - public Map getMarkings() { + public Markings getMarkings() { return this.markings; } - public void setMarkings(Map markings) { + public void setMarkings(Markings markings) { this.markings = markings; } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/VertexValue.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/VertexValue.java index bfbf4614323..6d8c1df3010 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/VertexValue.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/VertexValue.java @@ -1,8 +1,7 @@ package datawave.ingest.mapreduce.handler.edge.define; -import java.util.Map; - import datawave.ingest.data.config.NormalizedContentInterface; +import datawave.marking.Markings; /** * Combines a VertexDefinition with value obtained from a {@link datawave.ingest.data.config.NormalizedContentInterface}. @@ -23,7 +22,7 @@ public enum ValueType { private String indexedRealmLabel = null; private String eventRealmLabel = null; private String sourceIndex = null; - private Map markings = null; + private Markings markings = null; private String maskedValue = null; private boolean hasMaskedValue = false; private String relationshipType = null; @@ -50,7 +49,7 @@ public VertexValue(boolean useRealm, String indexedRealmLabel, String eventRealm } } - public Map getMarkings() { + public Markings getMarkings() { return this.markings; } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorDataTypeHandler.java index 0abad1bf986..841e4e1f4df 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorDataTypeHandler.java @@ -1,9 +1,10 @@ package datawave.ingest.mapreduce.handler.error; +import static java.nio.charset.StandardCharsets.UTF_8; + import java.io.IOException; import java.io.PrintStream; import java.util.Date; -import java.util.Map; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; @@ -19,6 +20,7 @@ import com.google.common.collect.Multimap; +import datawave.core.common.util.AccessExpressionUtil; import datawave.data.hash.UID; import datawave.data.hash.UIDBuilder; import datawave.ingest.config.IngestConfiguration; @@ -134,10 +136,9 @@ public void setup(TaskAttemptContext context) { this.conf = context.getConfiguration(); try { - Map defaultMarkings = markingsHelper.getDefaultMarkings(); - defaultVisibility = flatten(markingFunctions.translateToColumnVisibility(defaultMarkings)); + defaultVisibility = markingFunctions.toAccessExpression(markingsHelper.getDefaultMarkings()).getExpression().getBytes(UTF_8); } catch (MarkingFunctions.Exception e) { - throw new IllegalArgumentException("Failed to convert default markings to a ColumnVisibility.", e); + throw new IllegalArgumentException("Failed to convert default markings to an AccessExpression.", e); } // Initialize a UID builder based on the configuration @@ -317,11 +318,12 @@ private Key createKey(String row, Text colf, Text colq, byte[] vis, long ts) { * the value holding the markings * @return the visibility */ + @SuppressWarnings("unchecked") protected byte[] getVisibility(RawRecordContainer event, NormalizedContentInterface value) { byte[] visibility; if (value != null && value.getMarkings() != null && !value.getMarkings().isEmpty()) { try { - visibility = flatten(markingFunctions.translateToColumnVisibility(value.getMarkings())); + visibility = markingFunctions.toAccessExpression(value.getMarkings()).getExpression().getBytes(UTF_8); } catch (MarkingFunctions.Exception e) { log.error("Failed to create visibility from markings, using default", e); visibility = defaultVisibility; @@ -335,14 +337,14 @@ protected byte[] getVisibility(RawRecordContainer event, NormalizedContentInterf } /** - * Create a flattened visibility, using the cache if possible + * Normalize a ColumnVisibility expression via AccessExpression and return the expression bytes. * * @param vis * the visibility - * @return the flattened visibility + * @return the normalized visibility bytes */ protected byte[] flatten(ColumnVisibility vis) { - return markingFunctions.flatten(vis); + return MarkingFunctions.normalize(AccessExpressionUtil.toAccessExpression(vis)).getExpression().getBytes(UTF_8); } @Override diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java index 6f416d02fff..984d80fadd4 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java @@ -1,5 +1,7 @@ package datawave.ingest.mapreduce.handler.error; +import static java.nio.charset.StandardCharsets.UTF_8; + import java.io.IOException; import java.io.PrintStream; import java.util.ArrayList; @@ -156,7 +158,7 @@ public void setup(TaskAttemptContext context) { setShardDictionaryIndexTableName(tableName == null ? null : new Text(tableName)); try { - defaultVisibility = flatten(markingFunctions.translateToColumnVisibility(markingsHelper.getDefaultMarkings())); + defaultVisibility = markingFunctions.toAccessExpression(markingsHelper.getDefaultMarkings()).getExpression().getBytes(UTF_8); } catch (Exception e) { throw new IllegalArgumentException("Failed to parse security marking configuration", e); } @@ -342,12 +344,13 @@ public static void getStackTrace(DataOutputBuffer buffer, Throwable e) { * the value * @return the visibility */ + @SuppressWarnings("unchecked") @Override public byte[] getVisibility(RawRecordContainer event, NormalizedContentInterface value) { byte[] visibility; if (value != null && value.getMarkings() != null && !value.getMarkings().isEmpty()) { try { - visibility = flatten(markingFunctions.translateToColumnVisibility(value.getMarkings())); + visibility = markingFunctions.toAccessExpression(value.getMarkings()).getExpression().getBytes(UTF_8); } catch (MarkingFunctions.Exception e) { log.error("Failed to create visibility from markings, using default", e); visibility = defaultVisibility; diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/facet/FacetHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/facet/FacetHandler.java index 5e508dea644..75f14794c36 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/facet/FacetHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/facet/FacetHandler.java @@ -1,5 +1,7 @@ package datawave.ingest.mapreduce.handler.facet; +import static java.nio.charset.StandardCharsets.UTF_8; + import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; @@ -33,6 +35,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Multimap; +import datawave.core.common.util.AccessExpressionUtil; import datawave.ingest.data.RawRecordContainer; import datawave.ingest.data.Type; import datawave.ingest.data.TypeRegistry; @@ -192,7 +195,7 @@ public RawRecordMetadata getMetadata() { } protected byte[] flatten(ColumnVisibility vis) { - return markingFunctions == null ? vis.flatten() : markingFunctions.flatten(vis); + return MarkingFunctions.normalize(AccessExpressionUtil.toAccessExpression(vis)).getExpression().getBytes(UTF_8); } @Override diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java index 17a59dc6bd5..c75ebd27173 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java @@ -1,5 +1,7 @@ package datawave.ingest.mapreduce.handler.shard; +import static java.nio.charset.StandardCharsets.UTF_8; + import java.util.ArrayList; import java.util.Arrays; import java.util.BitSet; @@ -32,6 +34,7 @@ import com.google.common.hash.BloomFilter; import datawave.core.common.logging.ThreadConfigurableLogger; +import datawave.core.common.util.AccessExpressionUtil; import datawave.ingest.config.IngestConfiguration; import datawave.ingest.config.IngestConfigurationFactory; import datawave.ingest.data.RawRecordContainer; @@ -50,6 +53,7 @@ import datawave.ingest.util.BloomFilterWrapper; import datawave.ingest.util.DiskSpaceStarvationStrategy; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.query.model.Direction; import datawave.util.CompositeTimestamp; import datawave.util.TableName; @@ -1127,27 +1131,28 @@ public int getOffsetForYearIndex(String shard) { * the entry value * @return the visibility */ + @SuppressWarnings("unchecked") public byte[] getVisibility(RawRecordContainer event, NormalizedContentInterface value) { - ColumnVisibility visibility = event.getVisibility(); - if (value.getMarkings() != null && !value.getMarkings().isEmpty()) { + Markings markings = value.getMarkings(); + if (markings != null && !markings.isEmpty()) { try { - visibility = markingFunctions.translateToColumnVisibility(value.getMarkings()); + return markingFunctions.toAccessExpression(markings).getExpression().getBytes(UTF_8); } catch (MarkingFunctions.Exception e) { throw new RuntimeException("Cannot convert record-level markings into a column visibility", e); } } - return flatten(visibility); + return flatten(event.getVisibility()); } /** - * Create a flattened visibility, using the cache if possible + * Normalize a ColumnVisibility expression via AccessExpression and return the expression bytes. * * @param vis * the visibility - * @return the flattened visibility + * @return the normalized visibility bytes */ protected byte[] flatten(ColumnVisibility vis) { - return markingFunctions == null ? vis.flatten() : markingFunctions.flatten(vis); + return MarkingFunctions.normalize(AccessExpressionUtil.toAccessExpression(vis)).getExpression().getBytes(UTF_8); } /** diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/data/config/MarkingsHelperTest.java b/warehouse/ingest-core/src/test/java/datawave/ingest/data/config/MarkingsHelperTest.java index f4e90697c08..788863a0ecd 100644 --- a/warehouse/ingest-core/src/test/java/datawave/ingest/data/config/MarkingsHelperTest.java +++ b/warehouse/ingest-core/src/test/java/datawave/ingest/data/config/MarkingsHelperTest.java @@ -1,13 +1,12 @@ package datawave.ingest.data.config; -import java.util.Map; - import org.apache.hadoop.conf.Configuration; import org.junit.Assert; import org.junit.Test; import datawave.ingest.data.Type; -import datawave.marking.MarkingFunctions; +import datawave.marking.AccessExpressionMarkings; +import datawave.marking.Markings; public class MarkingsHelperTest { public static final String FIELD_NAME = "FIELDNAME1"; @@ -36,10 +35,10 @@ public void honorSpecifiedDatatypeFieldMarkings() { // use the same type name to retrieve a markings helper and retrieve the field marking MarkingsHelper markingsHelper = new MarkingsHelper.NoOp(conf, createType(typeName)); - Map fieldMarkingMap = markingsHelper.getFieldMarking(FIELD_NAME); + Markings fieldMarkings = markingsHelper.getFieldMarking(FIELD_NAME); - Assert.assertNotNull(fieldMarkingMap); - Assert.assertEquals(FIELD_MARKING_VALUE, fieldMarkingMap.get(MarkingFunctions.Default.COLUMN_VISIBILITY)); + Assert.assertNotNull(fieldMarkings); + Assert.assertEquals(FIELD_MARKING_VALUE, ((AccessExpressionMarkings) fieldMarkings).getAccessExpression().getExpression()); } private Configuration createConfWithFieldMarking(String typeName) { diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/data/config/NormalizedFieldAndValueTest.java b/warehouse/ingest-core/src/test/java/datawave/ingest/data/config/NormalizedFieldAndValueTest.java index f51e3eec718..e821d180e5c 100644 --- a/warehouse/ingest-core/src/test/java/datawave/ingest/data/config/NormalizedFieldAndValueTest.java +++ b/warehouse/ingest-core/src/test/java/datawave/ingest/data/config/NormalizedFieldAndValueTest.java @@ -1,11 +1,10 @@ package datawave.ingest.data.config; -import java.util.Map; - import org.junit.Assert; import org.junit.Test; import datawave.data.type.Type; +import datawave.marking.Markings; public class NormalizedFieldAndValueTest { @@ -19,7 +18,7 @@ public static class NonGroupedInstance implements NormalizedContentInterface { private String _eventFieldName; private String _eventFieldValue; - private Map _markings; + private Markings _markings; private Throwable _error; protected NonGroupedInstance() { @@ -84,7 +83,7 @@ public void setIndexedFieldValue(String val) { } @Override - public void setMarkings(Map markings) { + public void setMarkings(Markings markings) { _markings = markings; } @@ -95,7 +94,7 @@ public void setError(Throwable e) { } @Override - public Map getMarkings() { + public Markings getMarkings() { return _markings; } @@ -135,7 +134,7 @@ public static class GroupedInstance implements GroupedNormalizedContentInterface private String _eventFieldName; private String _eventFieldValue; - private Map _markings; + private Markings _markings; private Throwable _error; private boolean _grouped; @@ -208,7 +207,7 @@ public void setIndexedFieldValue(String val) { } @Override - public void setMarkings(Map markings) { + public void setMarkings(Markings markings) { _markings = markings; } @@ -219,7 +218,7 @@ public void setError(Throwable e) { } @Override - public Map getMarkings() { + public Markings getMarkings() { return _markings; } diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/data/config/ingest/FakeIngestHelper.java b/warehouse/ingest-core/src/test/java/datawave/ingest/data/config/ingest/FakeIngestHelper.java index 7fe022005c0..35858658c1b 100644 --- a/warehouse/ingest-core/src/test/java/datawave/ingest/data/config/ingest/FakeIngestHelper.java +++ b/warehouse/ingest-core/src/test/java/datawave/ingest/data/config/ingest/FakeIngestHelper.java @@ -1,7 +1,6 @@ package datawave.ingest.data.config.ingest; -import java.util.HashMap; -import java.util.Map; +import org.apache.accumulo.access.Access; import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; @@ -9,6 +8,7 @@ import datawave.ingest.data.RawRecordContainer; import datawave.ingest.data.config.NormalizedContentInterface; import datawave.ingest.data.config.NormalizedFieldAndValue; +import datawave.marking.AccessExpressionMarkings; /** * @@ -19,8 +19,7 @@ public class FakeIngestHelper extends BaseIngestHelper { @Override public Multimap getEventFields(RawRecordContainer event) { Multimap eventFields = HashMultimap.create(); - Map markings = new HashMap<>(); - markings.put("P", "PERSONAL"); + AccessExpressionMarkings markings = AccessExpressionMarkings.builder().accessExpression(Access.builder().build().newExpression("PERSONAL")).build(); eventFields.put("FAKE_FIELD", new NormalizedFieldAndValue("FAKE_FIELD", "fake value", markings)); return eventFields; diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/SimpleRawRecord.java b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/SimpleRawRecord.java index 9d8180b4021..e48309db946 100644 --- a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/SimpleRawRecord.java +++ b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/SimpleRawRecord.java @@ -9,8 +9,9 @@ import java.util.Date; import java.util.HashMap; import java.util.Map; -import java.util.TreeMap; +import org.apache.accumulo.access.Access; +import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.hadoop.io.Writable; @@ -20,6 +21,8 @@ import datawave.ingest.data.RawRecordContainer; import datawave.ingest.data.Type; import datawave.ingest.data.TypeRegistry; +import datawave.marking.AccessExpressionMarkings; +import datawave.marking.Markings; import datawave.util.CompositeTimestamp; /** @@ -27,9 +30,10 @@ */ public class SimpleRawRecord implements RawRecordContainer, Writable { + private static final Access ACCESS = Access.builder().build(); private UIDBuilder uidBuilder = HashUID.builder(); - private Map securityMarkings = new TreeMap<>(); + private Markings securityMarkings = null; private UID id = uidBuilder.newId(); private Type dataType; private long timestamp = CompositeTimestamp.INVALID_TIMESTAMP; @@ -45,23 +49,30 @@ public class SimpleRawRecord implements RawRecordContainer, Writable { private boolean fatalError = false; @Override - public Map getSecurityMarkings() { + public Markings getSecurityMarkings() { return securityMarkings; } @Override - public void setSecurityMarkings(Map securityMarkings) { + public void setSecurityMarkings(Markings securityMarkings) { this.securityMarkings = securityMarkings; } @Override public void addSecurityMarking(String domain, String marking) { - securityMarkings.put(domain, marking); + this.securityMarkings = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression(marking)).build(); } @Override public boolean hasSecurityMarking(String domain, String marking) { - return marking.equals(securityMarkings.get(domain)); + if (securityMarkings == null) { + return false; + } + Object m = securityMarkings.getMarkings(); + if (m instanceof AccessExpression) { + return ((AccessExpression) m).getExpression().equals(marking); + } + return false; } @Override @@ -244,7 +255,14 @@ public long getDataOutputSize() { @Override public void write(DataOutput dataOutput) throws IOException { - TestWritableUtil.writeMap(securityMarkings, dataOutput); + String expr = ""; + if (securityMarkings != null) { + Object m = securityMarkings.getMarkings(); + if (m instanceof AccessExpression) { + expr = ((AccessExpression) m).getExpression(); + } + } + dataOutput.writeUTF(expr); id.write(dataOutput); dataOutput.writeUTF(dataType.typeName()); @@ -269,7 +287,12 @@ public void write(DataOutput dataOutput) throws IOException { @Override public void readFields(DataInput dataInput) throws IOException { - securityMarkings = TestWritableUtil.readMap(dataInput); + String expr = dataInput.readUTF(); + if (!expr.isEmpty()) { + securityMarkings = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression(expr)).build(); + } else { + securityMarkings = null; + } id = uidBuilder.newId(); id.readFields(dataInput); diff --git a/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java b/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java index f09c886b623..47beef190e7 100644 --- a/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java +++ b/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java @@ -1,10 +1,10 @@ package datawave.ingest.csv.mr.input; import java.io.IOException; -import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; +import org.apache.accumulo.access.AccessExpression; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Text; @@ -27,8 +27,8 @@ import datawave.ingest.input.reader.event.EventFixer; import datawave.ingest.metadata.id.MetadataIdParser; import datawave.ingest.validation.EventValidator; +import datawave.marking.AccessExpressionMarkings; import datawave.marking.MarkingFunctions; -import datawave.marking.MarkingFunctionsFactory; public class CSVRecordReader extends CSVReaderBase implements EventFixer { @@ -42,7 +42,7 @@ public class CSVRecordReader extends CSVReaderBase implements EventFixer { private ExtendedCSVHelper csvHelper; private DataTypeOverrideHelper dataTypeHelper; - private Map securityMarkings; + private String securityMarkingExpression; @Override public void initialize(InputSplit genericSplit, TaskAttemptContext context) throws IOException { @@ -116,7 +116,7 @@ protected ExtendedCSVHelper createHelper(Configuration conf) { @Override public RawRecordContainer getEvent() { - securityMarkings = null; + securityMarkingExpression = null; csvEventId = null; metadataForValidation.clear(); return super.getEvent(); @@ -124,12 +124,12 @@ public RawRecordContainer getEvent() { @Override protected void decorateEvent() { - if (null != this.securityMarkings && !this.securityMarkings.isEmpty()) { - event.setSecurityMarkings(securityMarkings); + if (securityMarkingExpression != null) { try { - event.setVisibility(MarkingFunctionsFactory.createMarkingFunctions().translateToColumnVisibility(securityMarkings)); - } catch (MarkingFunctions.Exception e) { - log.error("Could not set default ColumnVisibility for the event", e); + AccessExpression ae = MarkingFunctions.normalize(MarkingFunctions.Default.ACCESS.newExpression(securityMarkingExpression)); + event.setSecurityMarkings(AccessExpressionMarkings.builder().accessExpression(ae).build()); + } catch (Exception e) { + log.error("Could not set security markings for the event", e); throw new RuntimeException(e); } } @@ -177,11 +177,12 @@ protected void checkField(String fieldName, String fieldValue) { // then put the marking value into this.securityMarkings, where the key is the field name for the marking // (as configured by EVENT_SECURITY_MARKING_FIELD_DOMAINS) if (this.csvHelper.getSecurityMarkingFieldDomainMap().containsKey(fieldName)) { - if (null == this.securityMarkings) { - this.securityMarkings = new HashMap<>(); - } if (!StringUtils.isEmpty(fieldValue)) { - this.securityMarkings.put(this.csvHelper.getSecurityMarkingFieldDomainMap().get(fieldName), fieldValue); + if (securityMarkingExpression == null) { + securityMarkingExpression = fieldValue; + } else { + securityMarkingExpression = "(" + securityMarkingExpression + ")&(" + fieldValue + ")"; + } } } // Now lets add metadata extracted from the parsers diff --git a/warehouse/ingest-csv/src/test/java/datawave/ingest/csv/NormalizedContentInterfaceTest.java b/warehouse/ingest-csv/src/test/java/datawave/ingest/csv/NormalizedContentInterfaceTest.java index cee757075ed..ca875cbf8be 100644 --- a/warehouse/ingest-csv/src/test/java/datawave/ingest/csv/NormalizedContentInterfaceTest.java +++ b/warehouse/ingest-csv/src/test/java/datawave/ingest/csv/NormalizedContentInterfaceTest.java @@ -29,6 +29,8 @@ import datawave.ingest.data.config.NormalizedFieldAndValue; import datawave.ingest.data.config.ingest.BaseIngestHelper; import datawave.ingest.data.config.ingest.IngestHelperInterface; +import datawave.marking.ColumnVisibilitySecurityMarking; +import datawave.marking.Markings; public class NormalizedContentInterfaceTest { private static final Logger log = LoggerFactory.getLogger(NormalizedContentInterfaceTest.class); @@ -178,10 +180,10 @@ private NormalizedContentInterface createNormalizedContent(String field, String return fieldAndValue; } - private Map createDefaultMarkings() { - Map defaultMarkings = new HashMap<>(1); - defaultMarkings.put("columnVisibility", "PRIVATE"); - return defaultMarkings; + private Markings createDefaultMarkings() { + ColumnVisibilitySecurityMarking sm = new ColumnVisibilitySecurityMarking(); + sm.setColumnVisibility("PRIVATE"); + return sm.toMarkings(); } private void compare(Multimap actualValues, Multimap expectedValues) { diff --git a/warehouse/ingest-ssdeep/src/main/java/datawave/ingest/mapreduce/handler/ssdeep/SSDeepIndexHandler.java b/warehouse/ingest-ssdeep/src/main/java/datawave/ingest/mapreduce/handler/ssdeep/SSDeepIndexHandler.java index d08dfd5667b..f91f8be1eb1 100644 --- a/warehouse/ingest-ssdeep/src/main/java/datawave/ingest/mapreduce/handler/ssdeep/SSDeepIndexHandler.java +++ b/warehouse/ingest-ssdeep/src/main/java/datawave/ingest/mapreduce/handler/ssdeep/SSDeepIndexHandler.java @@ -8,7 +8,6 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Text; @@ -135,10 +134,6 @@ public RawRecordMetadata getMetadata() { return null; } - protected byte[] flatten(ColumnVisibility vis) { - return markingFunctions == null ? vis.flatten() : markingFunctions.flatten(vis); - } - @Override public long process(KEYIN key, RawRecordContainer event, Multimap fields, TaskInputOutputContext context, ContextWriter contextWriter) diff --git a/warehouse/query-core/src/main/java/datawave/core/iterators/FieldIndexCountingIteratorPerVisibility.java b/warehouse/query-core/src/main/java/datawave/core/iterators/FieldIndexCountingIteratorPerVisibility.java index 9fa11756fe3..b96a188f1ae 100644 --- a/warehouse/query-core/src/main/java/datawave/core/iterators/FieldIndexCountingIteratorPerVisibility.java +++ b/warehouse/query-core/src/main/java/datawave/core/iterators/FieldIndexCountingIteratorPerVisibility.java @@ -33,6 +33,7 @@ import com.google.common.collect.Sets; +import datawave.core.common.util.AccessExpressionUtil; import datawave.iterators.IteratorSettingHelper; import datawave.marking.MarkingFunctions; import datawave.query.Constants; @@ -565,7 +566,8 @@ private Map buildReturnKeys() { ColumnVisibility cv = null; try { // Calculate the columnVisibility for this key from the combiner. - cv = MarkingFunctions.Factory.createMarkingFunctions().combine(columnVisibilities); + cv = AccessExpressionUtil.toColumnVisibility(MarkingFunctions.Factory.createMarkingFunctions().combine( + columnVisibilities.stream().map(AccessExpressionUtil::toAccessExpression).collect(java.util.stream.Collectors.toSet()))); } catch (Exception e) { log.error("Could not create combined columnVisibility for the count", e); return null; diff --git a/warehouse/query-core/src/main/java/datawave/core/iterators/GlobalIndexDateSummaryIterator.java b/warehouse/query-core/src/main/java/datawave/core/iterators/GlobalIndexDateSummaryIterator.java index 6329697cc99..c1852d12d8a 100644 --- a/warehouse/query-core/src/main/java/datawave/core/iterators/GlobalIndexDateSummaryIterator.java +++ b/warehouse/query-core/src/main/java/datawave/core/iterators/GlobalIndexDateSummaryIterator.java @@ -25,6 +25,7 @@ import com.google.common.collect.Sets; import com.google.protobuf.InvalidProtocolBufferException; +import datawave.core.common.util.AccessExpressionUtil; import datawave.ingest.protobuf.Uid; import datawave.ingest.protobuf.Uid.List.Builder; import datawave.marking.MarkingFunctions; @@ -256,7 +257,8 @@ public Map getKeyValues() throws IOException { Set columnVisibilities = this.columnVisibilitiesMap.get(datatype); // Note that the access controls found in the combined ColumnVisibility will be pulled out appropriately here - ColumnVisibility cv = MarkingFunctions.Factory.createMarkingFunctions().combine(columnVisibilities); + ColumnVisibility cv = AccessExpressionUtil.toColumnVisibility(MarkingFunctions.Factory.createMarkingFunctions().combine( + columnVisibilities.stream().map(AccessExpressionUtil::toAccessExpression).collect(java.util.stream.Collectors.toSet()))); // Create a new Key compatible with the shardIndex key format Key k = new Key(this.fieldValue, this.fieldName, this.date + '\0' + datatype, new String(cv.getExpression())); diff --git a/warehouse/query-core/src/main/java/datawave/core/iterators/ResultCountingIterator.java b/warehouse/query-core/src/main/java/datawave/core/iterators/ResultCountingIterator.java index 04027b8beac..bd537b7b90f 100644 --- a/warehouse/query-core/src/main/java/datawave/core/iterators/ResultCountingIterator.java +++ b/warehouse/query-core/src/main/java/datawave/core/iterators/ResultCountingIterator.java @@ -31,6 +31,7 @@ import com.google.common.cache.CacheBuilder; import com.google.common.collect.Sets; +import datawave.core.common.util.AccessExpressionUtil; import datawave.marking.MarkingFunctions; /** @@ -209,7 +210,8 @@ public Value getTopValue() { ColumnVisibility cv = null; try { - cv = MarkingFunctions.Factory.createMarkingFunctions().combine(columnVisibilities); + cv = AccessExpressionUtil.toColumnVisibility(MarkingFunctions.Factory.createMarkingFunctions() + .combine(columnVisibilities.stream().map(AccessExpressionUtil::toAccessExpression).collect(java.util.stream.Collectors.toSet()))); } catch (MarkingFunctions.Exception e) { log.error("Could not create combined columnVisibility for the count", e); return null; diff --git a/warehouse/query-core/src/main/java/datawave/query/attributes/Attribute.java b/warehouse/query-core/src/main/java/datawave/query/attributes/Attribute.java index 567d61e779a..a9d5f3a258a 100644 --- a/warehouse/query-core/src/main/java/datawave/query/attributes/Attribute.java +++ b/warehouse/query-core/src/main/java/datawave/query/attributes/Attribute.java @@ -8,6 +8,7 @@ import java.util.Arrays; import java.util.Collection; +import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.data.ArrayByteSequence; import org.apache.accumulo.core.data.ByteSequence; import org.apache.accumulo.core.data.Key; @@ -26,6 +27,7 @@ import datawave.core.cache.CaffeineClassCache; import datawave.core.cache.ClassCache; +import datawave.core.common.util.AccessExpressionUtil; import datawave.query.Constants; import datawave.query.jexl.DatawaveJexlContext; @@ -68,6 +70,15 @@ public ColumnVisibility getColumnVisibility() { return Constants.EMPTY_VISIBILITY; } + /** + * Get the access expression for this attribute, converted from the column visibility. + * + * @return the access expression + */ + public AccessExpression getAccessExpression() { + return AccessExpressionUtil.toAccessExpression(getColumnVisibility()); + } + /** * Get a copy byte array that backs the {@link ColumnVisibility}. This avoids the expensive parse call found in the default constructor for the * ColumnVisibility. diff --git a/warehouse/query-core/src/main/java/datawave/query/attributes/AttributeBag.java b/warehouse/query-core/src/main/java/datawave/query/attributes/AttributeBag.java index 130a91beb24..f65437ebe7c 100644 --- a/warehouse/query-core/src/main/java/datawave/query/attributes/AttributeBag.java +++ b/warehouse/query-core/src/main/java/datawave/query/attributes/AttributeBag.java @@ -2,14 +2,15 @@ import java.io.Serializable; import java.util.Collection; +import java.util.stream.Collectors; +import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.commons.lang.mutable.MutableLong; import org.apache.log4j.Logger; -import com.google.common.collect.Sets; - +import datawave.core.common.util.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.MarkingFunctions.Exception; @@ -78,11 +79,9 @@ private void updateMetadata() { } protected ColumnVisibility combineAndSetColumnVisibilities(Collection>> attributes) throws Exception { - Collection columnVisibilities = Sets.newHashSet(); - for (Attribute attr : attributes) { - columnVisibilities.add(attr.getColumnVisibility()); - } - return MarkingFunctions.Factory.createMarkingFunctions().combine(columnVisibilities); + Collection accessExpressions = attributes.stream().map(attr -> AccessExpressionUtil.toAccessExpression(attr.getColumnVisibility())) + .collect(Collectors.toSet()); + return AccessExpressionUtil.toColumnVisibility(MarkingFunctions.Factory.createMarkingFunctions().combine(accessExpressions)); } private long updateTimestamps() { diff --git a/warehouse/query-core/src/main/java/datawave/query/attributes/Document.java b/warehouse/query-core/src/main/java/datawave/query/attributes/Document.java index a3a76c51b0f..b398953957c 100644 --- a/warehouse/query-core/src/main/java/datawave/query/attributes/Document.java +++ b/warehouse/query-core/src/main/java/datawave/query/attributes/Document.java @@ -33,7 +33,9 @@ import com.google.common.collect.Iterators; import com.google.common.collect.Sets; +import datawave.core.common.util.AccessExpressionUtil; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.query.Constants; import datawave.query.collections.FunctionalSet; import datawave.query.composite.CompositeMetadata; @@ -84,12 +86,12 @@ public MarkingFunctions getMarkingFunctions() { return MarkingFunctions.Factory.createMarkingFunctions(); } - public Map getMarkings() { + public Markings getMarkings() { try { MarkingFunctions markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); - return markingFunctions.translateFromColumnVisibility(getColumnVisibility()); + return markingFunctions.fromAccessExpression(AccessExpressionUtil.toAccessExpression(getColumnVisibility())); } catch (MarkingFunctions.Exception e) {} - return Collections.emptyMap(); + return null; } public Document() { diff --git a/warehouse/query-core/src/main/java/datawave/query/common/grouping/AbstractAggregator.java b/warehouse/query-core/src/main/java/datawave/query/common/grouping/AbstractAggregator.java index 97f5a176a95..4fa5053d375 100644 --- a/warehouse/query-core/src/main/java/datawave/query/common/grouping/AbstractAggregator.java +++ b/warehouse/query-core/src/main/java/datawave/query/common/grouping/AbstractAggregator.java @@ -2,7 +2,7 @@ import java.util.Set; -import org.apache.accumulo.core.security.ColumnVisibility; +import org.apache.accumulo.access.AccessExpression; import datawave.query.attributes.Attribute; @@ -32,7 +32,7 @@ public String getField() { } @Override - public abstract Set getColumnVisibilities(); + public abstract Set getAccessExpressions(); @Override public abstract AGGREGATE getAggregation(); diff --git a/warehouse/query-core/src/main/java/datawave/query/common/grouping/Aggregator.java b/warehouse/query-core/src/main/java/datawave/query/common/grouping/Aggregator.java index 1b9329e6adc..70f8d60fc32 100644 --- a/warehouse/query-core/src/main/java/datawave/query/common/grouping/Aggregator.java +++ b/warehouse/query-core/src/main/java/datawave/query/common/grouping/Aggregator.java @@ -3,7 +3,7 @@ import java.util.Collection; import java.util.Set; -import org.apache.accumulo.core.security.ColumnVisibility; +import org.apache.accumulo.access.AccessExpression; import datawave.query.attributes.Attribute; @@ -30,11 +30,11 @@ public interface Aggregator { String getField(); /** - * Returns an unmodifiable set of all distinct column visibilities for each attribute aggregated into this aggregator. Possibly empty, but never null. + * Returns an unmodifiable set of all distinct access expressions for each attribute aggregated into this aggregator. Possibly empty, but never null. * - * @return a set of the column visibilities + * @return a set of the access expressions */ - Set getColumnVisibilities(); + Set getAccessExpressions(); /** * Return the aggregation result. diff --git a/warehouse/query-core/src/main/java/datawave/query/common/grouping/AverageAggregator.java b/warehouse/query-core/src/main/java/datawave/query/common/grouping/AverageAggregator.java index 976dfda7b96..161f47609ec 100644 --- a/warehouse/query-core/src/main/java/datawave/query/common/grouping/AverageAggregator.java +++ b/warehouse/query-core/src/main/java/datawave/query/common/grouping/AverageAggregator.java @@ -7,7 +7,7 @@ import java.util.HashSet; import java.util.Set; -import org.apache.accumulo.core.security.ColumnVisibility; +import org.apache.accumulo.access.AccessExpression; import org.apache.commons.lang.builder.ToStringBuilder; import datawave.query.attributes.Attribute; @@ -37,26 +37,26 @@ public class AverageAggregator extends AbstractAggregator { private BigDecimal average; /** - * The column visibilities of all attributes aggregated. + * The access expressions of all attributes aggregated. */ - private final Set columnVisibilities; + private final Set accessExpressions; public static AverageAggregator of(String field, TypeAttribute numerator, TypeAttribute divisor) { - return new AverageAggregator(field, numerator.getType().getDelegate(), divisor.getType().getDelegate(), numerator.getColumnVisibility()); + return new AverageAggregator(field, numerator.getType().getDelegate(), divisor.getType().getDelegate(), numerator.getAccessExpression()); } public AverageAggregator(String field) { super(field); - this.columnVisibilities = new HashSet<>(); + this.accessExpressions = new HashSet<>(); } - private AverageAggregator(String field, BigDecimal numerator, BigDecimal divisor, ColumnVisibility columnVisibility) { + private AverageAggregator(String field, BigDecimal numerator, BigDecimal divisor, AccessExpression expression) { this(field); this.numerator = numerator; this.divisor = divisor; this.average = numerator.divide(divisor, MATH_CONTEXT); - if (columnVisibility != null) { - this.columnVisibilities.add(columnVisibility); + if (expression != null) { + this.accessExpressions.add(expression); } } @@ -71,8 +71,8 @@ public AggregateOperation getOperation() { } @Override - public Set getColumnVisibilities() { - return Collections.unmodifiableSet(columnVisibilities); + public Set getAccessExpressions() { + return Collections.unmodifiableSet(accessExpressions); } /** @@ -129,7 +129,7 @@ public void aggregate(Attribute value) { divisor = divisor.add(BigDecimal.ONE); } average = numerator.divide(divisor, MATH_CONTEXT); - columnVisibilities.add(value.getColumnVisibility()); + accessExpressions.add(value.getAccessExpression()); } @Override @@ -139,7 +139,7 @@ public void merge(Aggregator other) { this.numerator = numerator.add(aggregator.numerator); this.divisor = divisor.add(aggregator.divisor); this.average = this.numerator.divide(this.divisor, MATH_CONTEXT); - this.columnVisibilities.addAll(aggregator.columnVisibilities); + this.accessExpressions.addAll(aggregator.accessExpressions); } else { throw new IllegalArgumentException("Cannot merge instance of " + other.getClass().getName()); } @@ -148,6 +148,6 @@ public void merge(Aggregator other) { @Override public String toString() { return new ToStringBuilder(this).append("field", field).append("average", average).append("numerator", numerator).append("divisor", divisor) - .append("columnVisibilities", columnVisibilities).toString(); + .append("accessExpressions", accessExpressions).toString(); } } diff --git a/warehouse/query-core/src/main/java/datawave/query/common/grouping/CountAggregator.java b/warehouse/query-core/src/main/java/datawave/query/common/grouping/CountAggregator.java index 1144d1f298f..dc4c4243c3f 100644 --- a/warehouse/query-core/src/main/java/datawave/query/common/grouping/CountAggregator.java +++ b/warehouse/query-core/src/main/java/datawave/query/common/grouping/CountAggregator.java @@ -5,7 +5,7 @@ import java.util.HashSet; import java.util.Set; -import org.apache.accumulo.core.security.ColumnVisibility; +import org.apache.accumulo.access.AccessExpression; import org.apache.commons.lang.builder.ToStringBuilder; import datawave.query.attributes.Attribute; @@ -22,24 +22,24 @@ public class CountAggregator extends AbstractAggregator { private long count; /** - * The column visibilities of all attributes aggregated. + * The access expressions of all attributes aggregated. */ - private final Set columnVisibilities; + private final Set accessExpressions; public static CountAggregator of(String field, TypeAttribute attribute) { - return new CountAggregator(field, attribute.getType().getDelegate().longValue(), attribute.getColumnVisibility()); + return new CountAggregator(field, attribute.getType().getDelegate().longValue(), attribute.getAccessExpression()); } public CountAggregator(String field) { super(field); - this.columnVisibilities = new HashSet<>(); + this.accessExpressions = new HashSet<>(); } - private CountAggregator(String field, long count, ColumnVisibility visibility) { + private CountAggregator(String field, long count, AccessExpression expression) { this(field); this.count = count; - if (visibility != null) { - columnVisibilities.add(visibility); + if (expression != null) { + accessExpressions.add(expression); } } @@ -54,8 +54,8 @@ public AggregateOperation getOperation() { } @Override - public Set getColumnVisibilities() { - return Collections.unmodifiableSet(columnVisibilities); + public Set getAccessExpressions() { + return Collections.unmodifiableSet(accessExpressions); } /** @@ -82,7 +82,7 @@ public boolean hasAggregation() { @Override public void aggregate(Attribute value) { count++; - this.columnVisibilities.add(value.getColumnVisibility()); + this.accessExpressions.add(value.getAccessExpression()); } @Override @@ -90,7 +90,7 @@ public void merge(Aggregator other) { if (other instanceof CountAggregator) { CountAggregator aggregator = (CountAggregator) other; this.count += aggregator.count; - this.columnVisibilities.addAll(aggregator.columnVisibilities); + this.accessExpressions.addAll(aggregator.accessExpressions); } else { throw new IllegalArgumentException("Cannot merge instance of " + other.getClass().getName()); } @@ -98,6 +98,6 @@ public void merge(Aggregator other) { @Override public String toString() { - return new ToStringBuilder(this).append("field", field).append("count", count).append("columnVisibilities", columnVisibilities).toString(); + return new ToStringBuilder(this).append("field", field).append("count", count).append("accessExpressions", accessExpressions).toString(); } } diff --git a/warehouse/query-core/src/main/java/datawave/query/common/grouping/DocumentGrouper.java b/warehouse/query-core/src/main/java/datawave/query/common/grouping/DocumentGrouper.java index b7131c34b65..838a16e71df 100644 --- a/warehouse/query-core/src/main/java/datawave/query/common/grouping/DocumentGrouper.java +++ b/warehouse/query-core/src/main/java/datawave/query/common/grouping/DocumentGrouper.java @@ -310,7 +310,7 @@ private void extractGroupsFromDocument() { // Create a new group and merge it into the existing groups. Group group = new Group(grouping, count); group.setFieldAggregator(fieldAggregator); - group.addDocumentVisibility(document.getColumnVisibility()); + group.addDocumentExpression(document.getAccessExpression()); groups.mergeOrPutGroup(group); } } @@ -630,9 +630,9 @@ private void trackGroup(Grouping grouping) { } // Add the visibilities of each attribute in the grouping for combination later, and increment the count for how many times this distinct // grouping was seen. - group.addAttributeVisibilities(grouping); + group.addAttributeExpressions(grouping); group.incrementCount(); - group.addDocumentVisibility(document.getColumnVisibility()); + group.addDocumentExpression(document.getAccessExpression()); } private Set> createGroupingAttributes(String field, Set> attributes) { diff --git a/warehouse/query-core/src/main/java/datawave/query/common/grouping/Group.java b/warehouse/query-core/src/main/java/datawave/query/common/grouping/Group.java index d6f1388ee34..9e028551d2f 100644 --- a/warehouse/query-core/src/main/java/datawave/query/common/grouping/Group.java +++ b/warehouse/query-core/src/main/java/datawave/query/common/grouping/Group.java @@ -4,7 +4,7 @@ import java.util.HashSet; import java.util.Set; -import org.apache.accumulo.core.security.ColumnVisibility; +import org.apache.accumulo.access.AccessExpression; import org.apache.commons.lang.builder.ToStringBuilder; import com.google.common.collect.HashMultimap; @@ -22,14 +22,14 @@ public class Group { private final ImmutableGrouping grouping; /** - * The different column visibilities seen for each attribute that makes up the grouping. + * The different access expressions seen for each attribute that makes up the grouping. */ - private final Multimap,ColumnVisibility> attributeVisibilities = HashMultimap.create(); + private final Multimap,AccessExpression> attributeExpressions = HashMultimap.create(); /** - * The column visibilities for each document that contributed entries to this grouping. + * The access expressions for each document that contributed entries to this grouping. */ - private final Set documentVisibilities = new HashSet<>(); + private final Set documentExpressions = new HashSet<>(); /** * The total number of times the distinct grouping was seen. @@ -47,7 +47,7 @@ public Group(Grouping grouping) { public Group(Grouping grouping, int count) { this.grouping = new ImmutableGrouping(grouping); - addAttributeVisibilities(this.grouping); + addAttributeExpressions(this.grouping); this.count = count; } @@ -70,45 +70,45 @@ public Grouping getGrouping() { } /** - * Add the column visibilities from each of the given attributes to the set of attribute visibilities for this group. + * Add the access expressions from each of the given attributes to the set of attribute expressions for this group. * * @param grouping - * the attributes to add visibilities from + * the attributes to add expressions from */ - public void addAttributeVisibilities(Grouping grouping) { + public void addAttributeExpressions(Grouping grouping) { for (GroupingAttribute attribute : grouping) { - attributeVisibilities.put(attribute, attribute.getColumnVisibility()); + attributeExpressions.put(attribute, attribute.getAccessExpression()); } } /** - * Return the set of column visibilities seen for the given attribute. + * Return the set of access expressions seen for the given attribute. * * @param attribute * the attribute - * @return the column visibilities seen for the given attributes + * @return the access expressions seen for the given attribute */ - public Collection getVisibilitiesForAttribute(GroupingAttribute attribute) { - return attributeVisibilities.get(attribute); + public Collection getAccessExpressionsForAttribute(GroupingAttribute attribute) { + return attributeExpressions.get(attribute); } /** - * Add the column visibility to the set of visibilities of documents for which we have seen the grouping of this group in. + * Add the access expression to the set of expressions of documents for which we have seen the grouping of this group in. * - * @param columnVisibility - * the visibility to add + * @param expression + * the expression to add */ - public void addDocumentVisibility(ColumnVisibility columnVisibility) { - this.documentVisibilities.add(columnVisibility); + public void addDocumentExpression(AccessExpression expression) { + this.documentExpressions.add(expression); } /** - * Return the set of all distinct column visibilities from documents that we have seen this group in. + * Return the set of all distinct access expressions from documents that we have seen this group in. * - * @return the document column visibilities + * @return the document access expressions */ - public Set getDocumentVisibilities() { - return documentVisibilities; + public Set getDocumentExpressions() { + return documentExpressions; } /** @@ -163,15 +163,15 @@ public void aggregateAll(String field, Collection fields) { * the group to merge */ public void merge(Group other) { - this.attributeVisibilities.putAll(other.attributeVisibilities); - this.documentVisibilities.addAll(other.documentVisibilities); + this.attributeExpressions.putAll(other.attributeExpressions); + this.documentExpressions.addAll(other.documentExpressions); this.count += other.count; this.fieldAggregator.merge(other.fieldAggregator); } @Override public String toString() { - return new ToStringBuilder(this).append("grouping", grouping).append("attributeVisibilities", attributeVisibilities) - .append("documentVisibilities", documentVisibilities).append("count", count).append("aggregatedFields", fieldAggregator).toString(); + return new ToStringBuilder(this).append("grouping", grouping).append("attributeExpressions", attributeExpressions) + .append("documentExpressions", documentExpressions).append("count", count).append("aggregatedFields", fieldAggregator).toString(); } } diff --git a/warehouse/query-core/src/main/java/datawave/query/common/grouping/GroupingUtils.java b/warehouse/query-core/src/main/java/datawave/query/common/grouping/GroupingUtils.java index 34375910c4a..5d20a47a6bf 100644 --- a/warehouse/query-core/src/main/java/datawave/query/common/grouping/GroupingUtils.java +++ b/warehouse/query-core/src/main/java/datawave/query/common/grouping/GroupingUtils.java @@ -8,12 +8,14 @@ import java.util.Map; import java.util.Set; +import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.security.ColumnVisibility; import org.slf4j.Logger; import com.google.common.base.Preconditions; +import datawave.core.common.util.AccessExpressionUtil; import datawave.data.type.NumberType; import datawave.marking.MarkingFunctions; import datawave.query.attributes.Document; @@ -31,25 +33,25 @@ public enum AverageAggregatorWriteFormat { private static final Logger log = getLogger(GroupingUtils.class); /** - * Returns a column visibility that results from the combination of all given visibilities using the given {@link MarkingFunctions}. + * Returns a column visibility that results from the combination of all given access expressions using the given {@link MarkingFunctions}. * - * @param visibilities - * the visibilities to combine + * @param expressions + * the access expressions to combine * @param markingFunctions - * the marking functions to combine the visibilities with + * the marking functions to combine the expressions with * @param failOnError - * if true and the visibilities cannot be combined, an {@link IllegalArgumentException} will be thrown. If false and the visibilities cannot be + * if true and the expressions cannot be combined, an {@link IllegalArgumentException} will be thrown. If false and the expressions cannot be * combined, it will be logged and a new, blank {@link ColumnVisibility} will be returned. * @return the combined column visibility */ - public static ColumnVisibility combineVisibilities(Collection visibilities, MarkingFunctions markingFunctions, boolean failOnError) { + public static ColumnVisibility combineVisibilities(Collection expressions, MarkingFunctions markingFunctions, boolean failOnError) { try { - return markingFunctions.combine(visibilities); + return AccessExpressionUtil.toColumnVisibility(markingFunctions.combine(expressions)); } catch (MarkingFunctions.Exception e) { if (failOnError) { - throw new IllegalArgumentException("Unable to combine visibilities: " + visibilities, e); + throw new IllegalArgumentException("Unable to combine access expressions: " + expressions, e); } else { - log.warn("Unable to combine visibilities from {}", visibilities); + log.warn("Unable to combine access expressions from {}", expressions); } } return new ColumnVisibility(); @@ -97,12 +99,12 @@ public static Document createDocument(Group group, Key key, MarkingFunctions mar Document document = new Document(key, true); // Set the visibility for the document to the combined visibility of each previous document in which this grouping was seen in. - document.setColumnVisibility(combineVisibilities(group.getDocumentVisibilities(), markingFunctions, true)); + document.setColumnVisibility(combineVisibilities(group.getDocumentExpressions(), markingFunctions, true)); // Add each of the grouping attributes to the document. for (GroupingAttribute attribute : group.getGrouping()) { // Update the visibility to the combined visibilities of each visibility seen for this attribute in a grouping. - attribute.setColumnVisibility(combineVisibilities(group.getVisibilitiesForAttribute(attribute), markingFunctions, false)); + attribute.setColumnVisibility(combineVisibilities(group.getAccessExpressionsForAttribute(attribute), markingFunctions, false)); document.put(attribute.getMetadata().getRow().toString(), attribute); } @@ -171,7 +173,7 @@ private static void addSumAggregation(Document document, String field, SumAggreg NumberType type = new NumberType(); type.setDelegate(aggregator.getAggregation()); TypeAttribute sumAttribute = new TypeAttribute<>(type, new Key(field + "_sum"), true); - sumAttribute.setColumnVisibility(combineVisibilities(aggregator.getColumnVisibilities(), markingFunctions, false)); + sumAttribute.setColumnVisibility(combineVisibilities(aggregator.getAccessExpressions(), markingFunctions, false)); document.put(field + DocumentGrouper.FIELD_SUM_SUFFIX, sumAttribute); } @@ -191,9 +193,9 @@ private static void addCountAggregation(Document document, String field, CountAg NumberType type = new NumberType(); type.setDelegate(BigDecimal.valueOf(aggregator.getAggregation())); TypeAttribute sumAttribute = new TypeAttribute<>(type, new Key(field + "_count"), true); - Set columnVisibilities = aggregator.getColumnVisibilities(); - if (!columnVisibilities.isEmpty()) { - sumAttribute.setColumnVisibility(combineVisibilities(aggregator.getColumnVisibilities(), markingFunctions, false)); + Set accessExpressions = aggregator.getAccessExpressions(); + if (!accessExpressions.isEmpty()) { + sumAttribute.setColumnVisibility(combineVisibilities(aggregator.getAccessExpressions(), markingFunctions, false)); } document.put(field + DocumentGrouper.FIELD_COUNT_SUFFIX, sumAttribute); } @@ -242,7 +244,7 @@ private static void addAverage(Document document, String field, AverageAggregato NumberType type = new NumberType(); type.setDelegate(aggregator.getAggregation()); TypeAttribute attribute = new TypeAttribute<>(type, new Key(field + "_average"), true); - attribute.setColumnVisibility(combineVisibilities(aggregator.getColumnVisibilities(), markingFunctions, false)); + attribute.setColumnVisibility(combineVisibilities(aggregator.getAccessExpressions(), markingFunctions, false)); document.put(field + DocumentGrouper.FIELD_AVERAGE_SUFFIX, attribute); } @@ -259,7 +261,7 @@ private static void addAverage(Document document, String field, AverageAggregato * the marking functions to use when combining column visibilities */ private static void addAverageNumeratorAndDivisor(Document document, String field, AverageAggregator aggregator, MarkingFunctions markingFunctions) { - ColumnVisibility visibility = combineVisibilities(aggregator.getColumnVisibilities(), markingFunctions, false); + ColumnVisibility visibility = combineVisibilities(aggregator.getAccessExpressions(), markingFunctions, false); // Add an attribute for the average's numerator. This is required to properly combine additional aggregations in future groupings. NumberType numeratorType = new NumberType(); diff --git a/warehouse/query-core/src/main/java/datawave/query/common/grouping/MaxAggregator.java b/warehouse/query-core/src/main/java/datawave/query/common/grouping/MaxAggregator.java index da9f538040e..435142c876e 100644 --- a/warehouse/query-core/src/main/java/datawave/query/common/grouping/MaxAggregator.java +++ b/warehouse/query-core/src/main/java/datawave/query/common/grouping/MaxAggregator.java @@ -3,7 +3,7 @@ import java.util.Collections; import java.util.Set; -import org.apache.accumulo.core.security.ColumnVisibility; +import org.apache.accumulo.access.AccessExpression; import org.apache.commons.lang.builder.ToStringBuilder; import datawave.query.attributes.Attribute; @@ -40,14 +40,14 @@ public AggregateOperation getOperation() { } /** - * Returns a singleton set containing the column visibility of the max attribute found. Possible empty, but never null. + * Returns a singleton set containing the access expression of the max attribute found. Possibly empty, but never null. * - * @return a set containing the column visibility + * @return a set containing the access expression */ @Override - public Set getColumnVisibilities() { + public Set getAccessExpressions() { if (max != null) { - return Collections.singleton(max.getColumnVisibility()); + return Collections.singleton(max.getAccessExpression()); } return Collections.emptySet(); } diff --git a/warehouse/query-core/src/main/java/datawave/query/common/grouping/MinAggregator.java b/warehouse/query-core/src/main/java/datawave/query/common/grouping/MinAggregator.java index 2a8d4a17316..7f652d0ba64 100644 --- a/warehouse/query-core/src/main/java/datawave/query/common/grouping/MinAggregator.java +++ b/warehouse/query-core/src/main/java/datawave/query/common/grouping/MinAggregator.java @@ -3,7 +3,7 @@ import java.util.Collections; import java.util.Set; -import org.apache.accumulo.core.security.ColumnVisibility; +import org.apache.accumulo.access.AccessExpression; import org.apache.commons.lang.builder.ToStringBuilder; import datawave.query.attributes.Attribute; @@ -43,14 +43,14 @@ public AggregateOperation getOperation() { } /** - * Returns a singleton set containing the column visibility of the min attribute found. Possible empty, but never null. + * Returns a singleton set containing the access expression of the min attribute found. Possibly empty, but never null. * - * @return a set containing the column visibility + * @return a set containing the access expression */ @Override - public Set getColumnVisibilities() { + public Set getAccessExpressions() { if (min != null) { - return Collections.singleton(min.getColumnVisibility()); + return Collections.singleton(min.getAccessExpression()); } return Collections.emptySet(); } diff --git a/warehouse/query-core/src/main/java/datawave/query/common/grouping/SumAggregator.java b/warehouse/query-core/src/main/java/datawave/query/common/grouping/SumAggregator.java index 888544af9fe..0e79304c5a3 100644 --- a/warehouse/query-core/src/main/java/datawave/query/common/grouping/SumAggregator.java +++ b/warehouse/query-core/src/main/java/datawave/query/common/grouping/SumAggregator.java @@ -5,7 +5,7 @@ import java.util.HashSet; import java.util.Set; -import org.apache.accumulo.core.security.ColumnVisibility; +import org.apache.accumulo.access.AccessExpression; import org.apache.commons.lang.builder.ToStringBuilder; import datawave.query.attributes.Attribute; @@ -23,25 +23,25 @@ public class SumAggregator extends AbstractAggregator { private BigDecimal sum; /** - * The column visibilities of all attributes aggregated. + * The access expressions of all attributes aggregated. */ - private final Set columnVisibilities; + private final Set accessExpressions; public static SumAggregator of(String field, TypeAttribute attribute) { BigDecimal sum = attribute.getType().getDelegate(); - return new SumAggregator(field, sum, attribute.getColumnVisibility()); + return new SumAggregator(field, sum, attribute.getAccessExpression()); } public SumAggregator(String field) { super(field); - this.columnVisibilities = new HashSet<>(); + this.accessExpressions = new HashSet<>(); } - private SumAggregator(String field, BigDecimal sum, ColumnVisibility visibility) { + private SumAggregator(String field, BigDecimal sum, AccessExpression expression) { this(field); this.sum = sum; - if (visibility != null) { - this.columnVisibilities.add(visibility); + if (expression != null) { + this.accessExpressions.add(expression); } } @@ -56,8 +56,8 @@ public AggregateOperation getOperation() { } @Override - public Set getColumnVisibilities() { - return Collections.unmodifiableSet(columnVisibilities); + public Set getAccessExpressions() { + return Collections.unmodifiableSet(accessExpressions); } /** @@ -96,7 +96,7 @@ public void aggregate(Attribute value) { } else { sum = sum.add(number); } - columnVisibilities.add(value.getColumnVisibility()); + accessExpressions.add(value.getAccessExpression()); } @Override @@ -104,7 +104,7 @@ public void merge(Aggregator other) { if (other instanceof SumAggregator) { SumAggregator aggregator = (SumAggregator) other; this.sum = this.sum.add(aggregator.sum); - this.columnVisibilities.addAll(aggregator.columnVisibilities); + this.accessExpressions.addAll(aggregator.accessExpressions); } else { throw new IllegalArgumentException("Cannot merge instance of " + other.getAggregation()); } @@ -112,6 +112,6 @@ public void merge(Aggregator other) { @Override public String toString() { - return new ToStringBuilder(this).append("field", field).append("sum", sum).append("columnVisibilities", columnVisibilities).toString(); + return new ToStringBuilder(this).append("field", field).append("sum", sum).append("accessExpressions", accessExpressions).toString(); } } diff --git a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java index 404d9c29dda..ecfa21136ca 100644 --- a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java +++ b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java @@ -30,6 +30,7 @@ import com.google.common.collect.Multimap; import com.google.protobuf.InvalidProtocolBufferException; +import datawave.core.common.util.AccessExpressionUtil; import datawave.ingest.protobuf.Uid; import datawave.marking.MarkingFunctions; import datawave.query.Constants; @@ -162,10 +163,12 @@ private DiscoveredThing aggregate(Collection termEntries) { } else { // Otherwise, combine the visibilities, and return the aggregated result. try { - ColumnVisibility visibility = markingFunctions.combine(visibilities); + ColumnVisibility visibility = AccessExpressionUtil.toColumnVisibility(markingFunctions + .combine(visibilities.stream().map(AccessExpressionUtil::toAccessExpression).collect(java.util.stream.Collectors.toSet()))); MapWritable countsByVis = new MapWritable(); visibilityToCounts.forEach((key, value) -> countsByVis.put(new Text(key), new LongWritable(value))); - return new DiscoveredThing(term, first.getField(), first.getDatatype(), date, new String(visibility.flatten()), count, countsByVis); + String normalizedVis = MarkingFunctions.normalize(AccessExpressionUtil.toAccessExpression(visibility)).getExpression(); + return new DiscoveredThing(term, first.getField(), first.getDatatype(), date, normalizedVis, count, countsByVis); } catch (Exception e) { if (log.isTraceEnabled()) { log.warn("Invalid column visibilities after combining " + visibilities); diff --git a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryTransformer.java index bbabc7c9537..5bd0978854a 100644 --- a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryTransformer.java @@ -1,7 +1,6 @@ package datawave.query.discovery; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -10,11 +9,13 @@ import com.google.common.base.Preconditions; +import datawave.core.common.util.AccessExpressionUtil; import datawave.core.query.cachedresults.CacheableLogic; import datawave.core.query.logic.BaseQueryLogic; import datawave.core.query.logic.BaseQueryLogicTransformer; import datawave.marking.MarkingFunctions; import datawave.marking.MarkingFunctions.Exception; +import datawave.marking.Markings; import datawave.microservice.query.Query; import datawave.query.model.QueryModel; import datawave.webservice.query.cachedresults.CacheableQueryRow; @@ -46,9 +47,9 @@ public EventBase transform(DiscoveredThing thing) { Preconditions.checkNotNull(thing, "Received a null object to transform!"); EventBase event = this.responseObjectFactory.getEvent(); - Map markings; + Markings markings; try { - markings = this.markingFunctions.translateFromColumnVisibility(new ColumnVisibility(thing.getColumnVisibility())); + markings = this.markingFunctions.fromAccessExpression(AccessExpressionUtil.toAccessExpression(new ColumnVisibility(thing.getColumnVisibility()))); } catch (Exception e) { throw new RuntimeException("could not parse to markings: " + thing.getColumnVisibility()); } @@ -68,8 +69,9 @@ public EventBase transform(DiscoveredThing thing) { if (thing.getCountsByColumnVisibility() != null && !thing.getCountsByColumnVisibility().isEmpty()) { for (Map.Entry entry : thing.getCountsByColumnVisibility().entrySet()) { try { - Map eMarkings = this.markingFunctions.translateFromColumnVisibility(new ColumnVisibility(entry.getKey().toString())); - fields.add(this.makeField("RECORD COUNT", new HashMap<>(), entry.getKey().toString(), 0L, entry.getValue().toString())); + Markings eMarkings = this.markingFunctions + .fromAccessExpression(AccessExpressionUtil.toAccessExpression(new ColumnVisibility(entry.getKey().toString()))); + fields.add(this.makeField("RECORD COUNT", eMarkings, entry.getKey().toString(), 0L, entry.getValue().toString())); } catch (Exception e) { throw new RuntimeException("could not parse to markings: " + thing.getColumnVisibility()); } @@ -92,7 +94,7 @@ public EventBase transform(DiscoveredThing thing) { return event; } - protected FieldBase makeField(String name, Map markings, String columnVisibility, Long timestamp, Object value) { + protected FieldBase makeField(String name, Markings markings, String columnVisibility, Long timestamp, Object value) { FieldBase field = this.responseObjectFactory.getField(); field.setName(name); field.setMarkings(markings); @@ -140,7 +142,7 @@ public Object readFromCache(CacheableQueryRow cacheableQueryRow) { if (this.variableFieldList == null) { this.variableFieldList = cacheableQueryRow.getVariableColumnNames(); } - Map markings = cacheableQueryRow.getMarkings(); + Markings markings = cacheableQueryRow.getMarkings(); String dataType = cacheableQueryRow.getDataType(); String internalId = cacheableQueryRow.getEventId(); String row = cacheableQueryRow.getRow(); @@ -163,7 +165,7 @@ public Object readFromCache(CacheableQueryRow cacheableQueryRow) { String columnValue = entry.getValue(); String columnVisibility = cacheableQueryRow.getColumnVisibility(columnName); Long columnTimestamp = cacheableQueryRow.getColumnTimestamp(columnName); - Map columnMarkings = cacheableQueryRow.getColumnMarkings(columnName); + Markings columnMarkings = cacheableQueryRow.getColumnMarkings(columnName); FieldBase field = this.responseObjectFactory.getField(); field.setName(columnName); field.setMarkings(columnMarkings); diff --git a/warehouse/query-core/src/main/java/datawave/query/discovery/TermInfoAggregation.java b/warehouse/query-core/src/main/java/datawave/query/discovery/TermInfoAggregation.java index 960a83e1889..a0292fcb03c 100644 --- a/warehouse/query-core/src/main/java/datawave/query/discovery/TermInfoAggregation.java +++ b/warehouse/query-core/src/main/java/datawave/query/discovery/TermInfoAggregation.java @@ -15,6 +15,7 @@ import com.google.common.base.Function; import com.google.common.collect.Sets; +import datawave.core.common.util.AccessExpressionUtil; import datawave.marking.MarkingFunctions; public class TermInfoAggregation implements Function,DiscoveredThing> { @@ -73,7 +74,8 @@ public DiscoveredThing apply(Collection from) { chosenCount = showReferenceCountInsteadOfTermCount ? referenceCount : termCount; try { - MarkingFunctions.Factory.createMarkingFunctions().translateFromColumnVisibility(ti.vis); // just to test parsing + MarkingFunctions.Factory.createMarkingFunctions().fromAccessExpression(AccessExpressionUtil.toAccessExpression(ti.vis)); // just to test + // parsing columnVisibilities.add(ti.vis); // Keep track of counts for individual vis @@ -107,7 +109,8 @@ public DiscoveredThing apply(Collection from) { ColumnVisibility columnVisibility = null; try { - columnVisibility = MarkingFunctions.Factory.createMarkingFunctions().combine(columnVisibilities); + columnVisibility = AccessExpressionUtil.toColumnVisibility(MarkingFunctions.Factory.createMarkingFunctions().combine( + columnVisibilities.stream().map(AccessExpressionUtil::toAccessExpression).collect(java.util.stream.Collectors.toSet()))); } catch (Exception e) { log.warn("Invalid columnvisibility after combining!", e); @@ -119,7 +122,8 @@ public DiscoveredThing apply(Collection from) { countsByVis.put(new Text(entry.getKey()), new VLongWritable(entry.getValue())); } - return new DiscoveredThing(term, field, type, date, new String(columnVisibility.flatten()), count, countsByVis); + return new DiscoveredThing(term, field, type, date, + MarkingFunctions.normalize(AccessExpressionUtil.toAccessExpression(columnVisibility)).getExpression(), count, countsByVis); } } } diff --git a/warehouse/query-core/src/main/java/datawave/query/iterator/GroupingIterator.java b/warehouse/query-core/src/main/java/datawave/query/iterator/GroupingIterator.java index 43928b68b77..02a7921170d 100644 --- a/warehouse/query-core/src/main/java/datawave/query/iterator/GroupingIterator.java +++ b/warehouse/query-core/src/main/java/datawave/query/iterator/GroupingIterator.java @@ -10,9 +10,9 @@ import java.util.Map; import java.util.Set; +import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.iterators.YieldCallback; -import org.apache.accumulo.core.security.ColumnVisibility; import org.slf4j.Logger; import com.google.common.collect.Maps; @@ -187,11 +187,11 @@ private Document flatten(List documents) { Document flattened = new Document(documents.get(documents.size() - 1).getMetadata(), true); int context = 0; - Set visibilities = new HashSet<>(); + Set expressions = new HashSet<>(); for (Document document : documents) { log.trace("document: {}", document); for (Map.Entry>> entry : document.entrySet()) { - visibilities.add(entry.getValue().getColumnVisibility()); + expressions.add(entry.getValue().getAccessExpression()); // Add a copy of each attribute to the flattened document with the context appended to the key, e.g. AGE becomes AGE.0. Attribute> attribute = entry.getValue(); attribute.setColumnVisibility(entry.getValue().getColumnVisibility()); @@ -203,8 +203,8 @@ private Document flatten(List documents) { context++; } - // Set the flattened document's visibility to the combined visibilities of each document. - flattened.setColumnVisibility(GroupingUtils.combineVisibilities(visibilities, markingFunctions, false)); + // Set the flattened document's visibility to the combined access expressions of each document. + flattened.setColumnVisibility(GroupingUtils.combineVisibilities(expressions, markingFunctions, false)); log.trace("flattened document: {}", flattened); return flattened; } diff --git a/warehouse/query-core/src/main/java/datawave/query/iterators/FieldIndexCountingIterator.java b/warehouse/query-core/src/main/java/datawave/query/iterators/FieldIndexCountingIterator.java index d6f11096c20..5e326f3df1f 100644 --- a/warehouse/query-core/src/main/java/datawave/query/iterators/FieldIndexCountingIterator.java +++ b/warehouse/query-core/src/main/java/datawave/query/iterators/FieldIndexCountingIterator.java @@ -29,6 +29,7 @@ import org.apache.hadoop.io.Text; import org.apache.log4j.Logger; +import datawave.core.common.util.AccessExpressionUtil; import datawave.core.common.util.TypeFilter; import datawave.iterators.IteratorSettingHelper; import datawave.marking.MarkingFunctions; @@ -587,7 +588,8 @@ private Key buildReturnKey() { } ColumnVisibility cv; try { - cv = MarkingFunctions.Factory.createMarkingFunctions().combine(columnVisibilities); + cv = AccessExpressionUtil.toColumnVisibility(MarkingFunctions.Factory.createMarkingFunctions() + .combine(columnVisibilities.stream().map(AccessExpressionUtil::toAccessExpression).collect(java.util.stream.Collectors.toSet()))); } catch (MarkingFunctions.Exception e) { log.error("Could not combine visibilities: " + visibilitySet + " " + e); return null; diff --git a/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java b/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java index 7cba3801a7f..95bf56ab54f 100644 --- a/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java +++ b/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java @@ -43,6 +43,7 @@ import com.google.common.collect.Maps; +import datawave.core.common.util.AccessExpressionUtil; import datawave.core.iterators.DatawaveFieldIndexListIteratorJexl; import datawave.marking.MarkingFunctions; import datawave.marking.MarkingFunctionsFactory; @@ -469,7 +470,8 @@ private void addHitsForFunction(String field, ASTFunctionNode node, HitListArith } try { - cv = MarkingFunctionsFactory.createMarkingFunctions().combine(cvs); + cv = AccessExpressionUtil.toColumnVisibility(MarkingFunctionsFactory.createMarkingFunctions() + .combine(cvs.stream().map(AccessExpressionUtil::toAccessExpression).collect(java.util.stream.Collectors.toSet()))); } catch (MarkingFunctions.Exception e) { log.error("Failed to combine column visibilities while generating HIT_TERM for phrase function for field [" + field + "]"); log.error("msg: ", e); diff --git a/warehouse/query-core/src/main/java/datawave/query/jexl/functions/IdentityAggregator.java b/warehouse/query-core/src/main/java/datawave/query/jexl/functions/IdentityAggregator.java index 026189ff52b..fee513e7ad7 100644 --- a/warehouse/query-core/src/main/java/datawave/query/jexl/functions/IdentityAggregator.java +++ b/warehouse/query-core/src/main/java/datawave/query/jexl/functions/IdentityAggregator.java @@ -11,7 +11,6 @@ import org.apache.accumulo.core.iterators.SortedKeyValueIterator; import org.apache.hadoop.io.Text; -import datawave.marking.ColumnVisibilityCache; import datawave.query.attributes.Attribute; import datawave.query.attributes.AttributeFactory; import datawave.query.attributes.Document; @@ -119,7 +118,7 @@ public Key apply(SortedKeyValueIterator itr, Document doc, AttributeF nextKey = (itr.hasTop() ? itr.getTopKey() : null); } - Key docKey = new Key(row, new Text(pointer.toArray()), new Text(), ColumnVisibilityCache.get(key.getColumnVisibilityData()), key.getTimestamp()); + Key docKey = new Key(row, new Text(pointer.toArray()), new Text(), key.getColumnVisibilityParsed(), key.getTimestamp()); Attribute attr = new DocumentKey(docKey, false); doc.put(Document.DOCKEY_FIELD_NAME, attr); diff --git a/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java b/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java index f7c95de1c41..668eba08655 100644 --- a/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java +++ b/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java @@ -20,6 +20,7 @@ import com.google.common.collect.Multimaps; import com.google.common.collect.Sets; +import datawave.core.common.util.AccessExpressionUtil; import datawave.data.type.OneToManyNormalizerType; import datawave.data.type.Type; import datawave.ingest.data.config.ingest.CompositeIngest; @@ -225,7 +226,7 @@ public Attribute getFieldValue(String fieldName, Key k) { } public Attribute joinAttributes(String compositeName, Collection> in, boolean isOverloadedComposite, String separator) throws Exception { - Collection columnVisibilities = Sets.newHashSet(); + Collection accessExpressions = Sets.newHashSet(); List dataList = new ArrayList<>(); long timestamp = 0; boolean toKeep = false; @@ -260,7 +261,7 @@ public Attribute joinAttributes(String compositeName, Collection dataList.add(base + value); } timestamp = Math.max(timestamp, attrs.getTimestamp()); - columnVisibilities.add(attrs.getColumnVisibility()); + accessExpressions.add(AccessExpressionUtil.toAccessExpression(attrs.getColumnVisibility())); } } } @@ -284,13 +285,13 @@ public Attribute joinAttributes(String compositeName, Collection } } timestamp = Math.max(timestamp, attr.getTimestamp()); - columnVisibilities.add(attr.getColumnVisibility()); + accessExpressions.add(AccessExpressionUtil.toAccessExpression(attr.getColumnVisibility())); } } if (log.isDebugEnabled()) { log.debug("dataList is " + dataList); } - ColumnVisibility combinedColumnVisibility = this.markingFunctions.combine(columnVisibilities); + ColumnVisibility combinedColumnVisibility = AccessExpressionUtil.toColumnVisibility(this.markingFunctions.combine(accessExpressions)); metadata = new Key(metadata.getRow(), metadata.getColumnFamily(), new Text(), combinedColumnVisibility, timestamp); if (dataList.size() == 1) { return this.attrFactory.create(compositeName, dataList.get(0), metadata, toKeep, true); diff --git a/warehouse/query-core/src/main/java/datawave/query/table/parser/EventKeyValueFactory.java b/warehouse/query-core/src/main/java/datawave/query/table/parser/EventKeyValueFactory.java index 60693cdefce..e977f9e4812 100644 --- a/warehouse/query-core/src/main/java/datawave/query/table/parser/EventKeyValueFactory.java +++ b/warehouse/query-core/src/main/java/datawave/query/table/parser/EventKeyValueFactory.java @@ -1,16 +1,13 @@ package datawave.query.table.parser; -import java.util.HashMap; -import java.util.Map; - import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.ColumnVisibility; -import com.google.common.collect.Maps; - +import datawave.core.common.util.AccessExpressionUtil; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.query.Constants; public class EventKeyValueFactory { @@ -44,7 +41,7 @@ public static EventKeyValue parse(Key key, Value value, Authorizations auths, Ma protected static void parseColumnVisibility(EventKeyValue event, Key key, Authorizations auths, MarkingFunctions markingFunctions) throws MarkingFunctions.Exception { - event.setMarkings(markingFunctions.translateFromColumnVisibilityForAuths(new ColumnVisibility(key.getColumnVisibility()), auths)); + event.setMarkings(markingFunctions.fromAccessExpression(AccessExpressionUtil.toAccessExpression(new ColumnVisibility(key.getColumnVisibility())))); } public static class EventKeyValue { @@ -53,7 +50,7 @@ public static class EventKeyValue { protected String uid = null; protected String fieldName = null; protected String fieldValue = null; - protected Map markings = null; + protected Markings markings = null; protected long timestamp = 0L; public String getShardId() { @@ -104,14 +101,12 @@ protected void setTimestamp(long timestamp) { this.timestamp = timestamp; } - public Map getMarkings() { - if (this.markings == null) - this.markings = Maps.newHashMap(); - return Maps.newHashMap(markings); + public Markings getMarkings() { + return this.markings; } - public void setMarkings(Map markings) { - this.markings = (markings == null ? new HashMap<>() : new HashMap<>(markings)); + public void setMarkings(Markings markings) { + this.markings = markings; } } diff --git a/warehouse/query-core/src/main/java/datawave/query/table/parser/TermFrequencyKeyValueFactory.java b/warehouse/query-core/src/main/java/datawave/query/table/parser/TermFrequencyKeyValueFactory.java index 466839c795a..80164dd823f 100644 --- a/warehouse/query-core/src/main/java/datawave/query/table/parser/TermFrequencyKeyValueFactory.java +++ b/warehouse/query-core/src/main/java/datawave/query/table/parser/TermFrequencyKeyValueFactory.java @@ -9,6 +9,7 @@ import com.google.protobuf.InvalidProtocolBufferException; +import datawave.core.common.util.AccessExpressionUtil; import datawave.ingest.protobuf.TermWeight; import datawave.marking.MarkingFunctions; import datawave.query.table.parser.EventKeyValueFactory.EventKeyValue; @@ -56,7 +57,7 @@ public static TermFrequencyKeyValue parse(Key key, Value value, Authorizations a protected static void parseColumnVisibility(TermFrequencyKeyValue tfkv, Key key, Authorizations auths, MarkingFunctions markingFunctions) throws MarkingFunctions.Exception { - tfkv.setMarkings(markingFunctions.translateFromColumnVisibilityForAuths(key.getColumnVisibilityParsed(), auths)); + tfkv.setMarkings(markingFunctions.fromAccessExpression(AccessExpressionUtil.toAccessExpression(key.getColumnVisibilityParsed()))); } public static class TermFrequencyKeyValue extends EventKeyValue { diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountAggregatingIterator.java b/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountAggregatingIterator.java index 08ec99cf1a7..49668cf37d3 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountAggregatingIterator.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountAggregatingIterator.java @@ -13,6 +13,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; @@ -25,6 +26,8 @@ import com.esotericsoftware.kryo.io.Input; import com.google.common.collect.Maps; +import datawave.core.common.util.AccessExpressionUtil; +import datawave.core.iterators.ResultCountingIterator; import datawave.marking.MarkingFunctions; import datawave.webservice.query.result.event.DefaultEvent; @@ -121,7 +124,7 @@ private Object getIntermediateEvent() { private static class CountEntryAggregator { private final AtomicLong count = new AtomicLong(0L); - private final Set cvs = new HashSet<>(); + private final Set columnVisibilities = new HashSet<>(); private final Transformer transformer; private final MarkingFunctions markingFunctions; @@ -136,13 +139,13 @@ public void addCount(long count) { } public void addColumnVisibility(ColumnVisibility cv) { - this.cvs.add(cv); + this.columnVisibilities.add(cv); } @SuppressWarnings("unchecked") public Object getAggregatedEvent() { - if (cvs.isEmpty() && count.get() == 0L) { - cvs.add(new ColumnVisibility("")); + if (columnVisibilities.isEmpty() && count.get() == 0L) { + columnVisibilities.add(new ColumnVisibility("")); } ColumnVisibility cv = getCombinedColumnVisibility(); @@ -151,7 +154,8 @@ public Object getAggregatedEvent() { private ColumnVisibility getCombinedColumnVisibility() { try { - return markingFunctions.combine(cvs); + return AccessExpressionUtil.toColumnVisibility(markingFunctions + .combine(columnVisibilities.stream().map(AccessExpressionUtil::toAccessExpression).collect(Collectors.toSet()))); } catch (Exception e) { log.error("Could not combine columnVisibilities for the count", e); return null; diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountResultPostprocessor.java b/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountResultPostprocessor.java index 5edb7a85dd5..ca3f16e4095 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountResultPostprocessor.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountResultPostprocessor.java @@ -11,6 +11,7 @@ import com.google.common.collect.Sets; +import datawave.core.common.util.AccessExpressionUtil; import datawave.core.query.logic.ResultPostprocessor; import datawave.marking.MarkingFunctions; import datawave.webservice.query.result.event.EventBase; @@ -70,7 +71,8 @@ public void apply(List results) { if (success) { ColumnVisibility columnVisibility = null; try { - columnVisibility = markingFunctions.combine(columnVisibilities); + columnVisibility = AccessExpressionUtil.toColumnVisibility(markingFunctions.combine( + columnVisibilities.stream().map(AccessExpressionUtil::toAccessExpression).collect(java.util.stream.Collectors.toSet()))); } catch (Exception e) { log.error("Could not create combined columnVisibilities for the count", e); } @@ -81,7 +83,7 @@ public void apply(List results) { // update the first result FieldBase countField = getCountField(firstResult.getFields()); countField.setValue(count); - countField.setColumnVisibility(columnVisibility); + countField.setColumnVisibility(new String(columnVisibility.getExpression())); } } } diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java index 261c988c016..19898771eb6 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java @@ -29,6 +29,7 @@ import org.apache.hadoop.io.Text; import org.apache.log4j.Logger; +import datawave.core.common.util.AccessExpressionUtil; import datawave.core.query.configuration.GenericQueryConfiguration; import datawave.core.query.logic.QueryLogicTransformer; import datawave.data.type.Type; @@ -507,7 +508,8 @@ public ColumnVisibility getColumnVisibility() { for (Text t : this.uniqueVisibilities) { columnVisibilities.add(new ColumnVisibility(t)); } - return tupleMarkingFunctions.combine(columnVisibilities); + return AccessExpressionUtil.toColumnVisibility(tupleMarkingFunctions.combine( + columnVisibilities.stream().map(AccessExpressionUtil::toAccessExpression).collect(java.util.stream.Collectors.toSet()))); } catch (MarkingFunctions.Exception e) { logger.error("Could not create combined column visibility for the count", e); diff --git a/warehouse/query-core/src/main/java/datawave/query/tld/TLDFieldIndexAggregator.java b/warehouse/query-core/src/main/java/datawave/query/tld/TLDFieldIndexAggregator.java index 06a7738edb6..5031c70acc3 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tld/TLDFieldIndexAggregator.java +++ b/warehouse/query-core/src/main/java/datawave/query/tld/TLDFieldIndexAggregator.java @@ -12,7 +12,6 @@ import org.apache.accumulo.core.iterators.SortedKeyValueIterator; import org.apache.hadoop.io.Text; -import datawave.marking.ColumnVisibilityCache; import datawave.query.Constants; import datawave.query.attributes.Attribute; import datawave.query.attributes.AttributeFactory; @@ -58,8 +57,7 @@ public Key apply(SortedKeyValueIterator itr, Document d, AttributeFac ByteSequence thisId = parseDatatypeUidFromFI(key.getColumnQualifierData()); if (docId == null || !docId.equals(thisId)) { docId = thisId; - Key docKey = new Key(key.getRow(), new Text(docId.toArray()), new Text(), ColumnVisibilityCache.get(key.getColumnVisibilityData()), - key.getTimestamp()); + Key docKey = new Key(key.getRow(), new Text(docId.toArray()), new Text(), key.getColumnVisibilityParsed(), key.getTimestamp()); attr = new DocumentKey(docKey, false); d.put(Document.DOCKEY_FIELD_NAME, attr); } diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java index 6d85f41df66..79360a8cfa7 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java @@ -2,7 +2,6 @@ import java.util.ArrayList; import java.util.Collection; -import java.util.Map; import java.util.Map.Entry; import org.apache.accumulo.core.data.Key; @@ -12,12 +11,14 @@ import com.google.common.base.Preconditions; +import datawave.core.common.util.AccessExpressionUtil; import datawave.core.query.exception.EmptyObjectException; import datawave.core.query.logic.BaseQueryLogic; import datawave.core.query.logic.Flushable; import datawave.core.query.logic.WritesQueryMetrics; import datawave.core.query.logic.WritesResultCardinalities; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.microservice.query.Query; import datawave.query.attributes.Document; import datawave.webservice.query.result.event.EventBase; @@ -165,7 +166,7 @@ private EventBase _transform(Entry documentEntry) throws EmptyObje protected EventBase buildResponse(Document document, Key documentKey, ColumnVisibility eventCV, String colf, String row, MarkingFunctions mf) throws MarkingFunctions.Exception { - Map markings = mf.translateFromColumnVisibility(eventCV); + Markings markings = mf.fromAccessExpression(AccessExpressionUtil.toAccessExpression(eventCV)); EventBase event = null; final Collection> documentFields = buildDocumentFields(documentKey, null, document, eventCV, mf); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformerSupport.java b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformerSupport.java index 35625d0d6dc..1846db653e5 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformerSupport.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformerSupport.java @@ -24,11 +24,14 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import datawave.core.common.util.AccessExpressionUtil; import datawave.core.query.exception.EmptyObjectException; import datawave.core.query.logic.BaseQueryLogic; import datawave.core.query.logic.WritesQueryMetrics; import datawave.core.query.logic.WritesResultCardinalities; +import datawave.marking.AccessExpressionMarkings; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.microservice.query.Query; import datawave.microservice.query.QueryImpl.Parameter; import datawave.microservice.querymetric.BaseQueryMetric; @@ -68,7 +71,7 @@ public abstract class DocumentTransformerSupport extends EventQueryTransfor protected Boolean reducedResponse; private static final Logger log = Logger.getLogger(DocumentTransformerSupport.class); - private static final Map EMPTY_MARKINGS = new HashMap<>(); + private static final Markings EMPTY_MARKINGS = AccessExpressionMarkings.builder().build(); private long sourceCount = 0; private long nextCount = 0; @@ -463,9 +466,8 @@ protected Collection> buildDocumentFields(Key documentKey, String f // Use the markings on the Field if we're returning the markings to the client if (!this.reducedResponse) { try { - Map markings = markingFunctions.translateFromColumnVisibility(attr.getColumnVisibility()); + Markings markings = markingFunctions.fromAccessExpression(AccessExpressionUtil.toAccessExpression(attr.getColumnVisibility())); FieldBase field = this.makeField(fieldName, markings, attr.getColumnVisibility(), attr.getTimestamp(), attr.getData()); - MarkingFunctions.Util.populate(field, markings); myFields.add(field); } catch (Exception ex) { log.error("unable to process markings:" + ex); @@ -495,8 +497,7 @@ protected Collection> buildDocumentFields(Key documentKey, String f * mapping of markings * @return a field */ - protected FieldBase createField(final String fieldName, final long ts, final Attribute attribute, Map markings, - String columnVisibility) { + protected FieldBase createField(final String fieldName, final long ts, final Attribute attribute, Markings markings, String columnVisibility) { if (markings == null || markings.isEmpty()) { log.warn("Null or empty markings for " + fieldName + ":" + attribute); } @@ -504,16 +505,14 @@ protected FieldBase createField(final String fieldName, final long ts, final return createField(fieldName, (Long) ts, attribute, markings, columnVisibility); } - protected FieldBase createField(final String fieldName, final Long ts, final Attribute attribute, Map markings, - String columnVisibility) { + protected FieldBase createField(final String fieldName, final Long ts, final Attribute attribute, Markings markings, String columnVisibility) { if (this.transformValuePrefixFields.contains(fieldName)) { return convertHitTermField(fieldName, ts, attribute, markings, columnVisibility); } return this.makeField(fieldName, markings, columnVisibility, ts, attribute.getData()); } - private FieldBase convertHitTermField(final String fieldName, final Long ts, Attribute attribute, Map markings, - String columnVisibility) { + private FieldBase convertHitTermField(final String fieldName, final Long ts, Attribute attribute, Markings markings, String columnVisibility) { return this.makeField(fieldName, markings, columnVisibility, ts, convertMappedAttribute(attribute).getData()); } diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformer.java index ee584c5733d..769cba260e7 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformer.java @@ -2,7 +2,6 @@ import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.Map.Entry; import org.apache.accumulo.core.data.Key; @@ -12,12 +11,14 @@ import com.google.protobuf.InvalidProtocolBufferException; +import datawave.core.common.util.AccessExpressionUtil; import datawave.core.query.cachedresults.CacheableLogic; import datawave.edge.model.EdgeModelFields; import datawave.edge.util.EdgeKey; import datawave.edge.util.EdgeValue; import datawave.edge.util.EdgeValueHelper; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.microservice.query.Query; import datawave.util.time.DateHelper; import datawave.webservice.query.result.edge.EdgeBase; @@ -40,7 +41,7 @@ public EdgeBase transform(Entry entry) { boolean statsEdge = edgeKey.isStatsKey(); try { - Map markings = markingFunctions.translateFromColumnVisibilityForAuths(new ColumnVisibility(edgeKey.getColvis()), auths); + Markings markings = markingFunctions.fromAccessExpression(AccessExpressionUtil.toAccessExpression(new ColumnVisibility(edgeKey.getColvis()))); edge.setMarkings(markings); edge.setEdgeType(edgeKey.getType()); edge.setEdgeRelationship(edgeKey.getRelationship()); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformerSupport.java b/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformerSupport.java index 5be63b09779..e5448a6584c 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformerSupport.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformerSupport.java @@ -6,9 +6,9 @@ import java.util.Map; import java.util.Set; +import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.Authorizations; -import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.builder.HashCodeBuilder; @@ -20,6 +20,7 @@ import datawave.edge.model.EdgeModelFields; import datawave.edge.util.EdgeValue; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.microservice.query.Query; import datawave.webservice.query.cachedresults.CacheableQueryRow; import datawave.webservice.query.exception.QueryException; @@ -99,16 +100,16 @@ public BaseQueryResponse createResponse(List resultList) { try { EdgeQueryResponseBase response = responseObjectFactory.getEdgeQueryResponse(); - Set uniqueColumnVisibilities = Sets.newHashSet(); + Set uniqueExpressions = Sets.newHashSet(); for (Object result : resultList) { EdgeBase edge = (EdgeBase) result; - Map markings = edge.getMarkings(); - uniqueColumnVisibilities.add(this.markingFunctions.translateToColumnVisibility(markings)); + Markings markings = edge.getMarkings(); + uniqueExpressions.add(this.markingFunctions.toAccessExpression(markings)); response.addEdge(edge); } - ColumnVisibility combinedVisibility = this.markingFunctions.combine(uniqueColumnVisibilities); - response.setMarkings(this.markingFunctions.translateFromColumnVisibility(combinedVisibility)); + AccessExpression combinedExpression = this.markingFunctions.combine(uniqueExpressions); + response.setMarkings(this.markingFunctions.fromAccessExpression(combinedExpression)); return response; } catch (Exception ex) { throw new RuntimeException("could not handle markings in resultList ", ex); @@ -171,7 +172,7 @@ public CacheableQueryRow writeToCache(Object o) throws QueryException { @Override public Object readFromCache(CacheableQueryRow cacheableQueryRow) { - Map markings = cacheableQueryRow.getMarkings(); + Markings markings = cacheableQueryRow.getMarkings(); EdgeBase edge = (EdgeBase) responseObjectFactory.getEdge(); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryDataDecorator.java b/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryDataDecorator.java index 640dc36b34c..f6155c3ef77 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryDataDecorator.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryDataDecorator.java @@ -9,6 +9,7 @@ import com.google.common.collect.Multimap; +import datawave.marking.Markings; import datawave.webservice.query.result.event.FieldBase; import datawave.webservice.query.result.event.ResponseObjectFactory; @@ -33,7 +34,7 @@ public void decorateData(Multimap data) { // multiple value source fields for the substitution value not supported -- use the first one Iterator collectionSourceItr = collectionSourceOfData.iterator(); FieldBase sourceOfData = collectionSourceItr.next(); - Map markings = sourceOfData.getMarkings(); + Markings markings = sourceOfData.getMarkings(); String id = sourceOfData.getValueString(); String newValue = entry.getValue().replace("@field_value@", id); @@ -61,7 +62,7 @@ public void decorateData(Multimap data) { } } - private FieldBase makeField(String name, Map markings, String columnVisibility, Long timestamp, Object value) { + private FieldBase makeField(String name, Markings markings, String columnVisibility, Long timestamp, Object value) { FieldBase field = this.responseObjectFactory.getField(); field.setName(name); field.setMarkings(markings); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java index 3b6eef77141..f8f4eeb42f2 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java @@ -5,20 +5,22 @@ import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; -import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.ColumnVisibility; import com.esotericsoftware.kryo.io.Input; +import datawave.core.common.util.AccessExpressionUtil; import datawave.core.query.cachedresults.CacheableLogic; import datawave.core.query.logic.BaseQueryLogic; import datawave.marking.MarkingFunctions; import datawave.marking.MarkingFunctions.Exception; +import datawave.marking.Markings; import datawave.microservice.query.Query; import datawave.query.Constants; import datawave.query.parser.EventFields; @@ -52,9 +54,9 @@ public EventBase transform(Entry entry) { } EventBase event = this.responseObjectFactory.getEvent(); - Map markings = null; + Markings markings = null; try { - markings = this.markingFunctions.translateFromColumnVisibilityForAuths(new ColumnVisibility(key.getColumnVisibility()), this.auths); + markings = this.markingFunctions.fromAccessExpression(AccessExpressionUtil.toAccessExpression(new ColumnVisibility(key.getColumnVisibility()))); } catch (Exception e) { log.error("could not translate " + key.getColumnVisibility() + " to markings, skipping entry"); return null; @@ -86,9 +88,9 @@ public EventBase transform(Entry entry) { String origFieldName = null; String fieldName = null; - // Hold unique Column Visibilities and merge them at the end - // for the overall event ColumnVisibility. - Set visibilitiesToMerge = new HashSet<>(); + // Hold unique AccessExpressions and merge them at the end + // for the overall event visibility. + Set expressionsToMerge = new HashSet<>(); for (Entry> e : eventFields.asMap().entrySet()) { origFieldName = e.getKey(); @@ -99,27 +101,28 @@ public EventBase transform(Entry entry) { } for (FieldValue fv : e.getValue()) { - visibilitiesToMerge.add(fv.getVisibility()); + AccessExpression fvExpression = AccessExpressionUtil.toAccessExpression(fv.getVisibility()); + expressionsToMerge.add(fvExpression); try { - Map fieldMarkings = this.markingFunctions.translateFromColumnVisibility(fv.getVisibility()); + Markings fieldMarkings = this.markingFunctions.fromAccessExpression(fvExpression); String value = new String(fv.getValue(), StandardCharsets.UTF_8); // if this is a content field name, then replace the value with the uid if (getContentFieldNames().contains(fieldName)) { value = baseUid; } - values.add(this.makeField(fieldName, fieldMarkings, new String(fv.getVisibility().getExpression()), entry.getKey().getTimestamp(), value)); + values.add(this.makeField(fieldName, fieldMarkings, fvExpression.getExpression(), entry.getKey().getTimestamp(), value)); } catch (Exception e1) { - throw new RuntimeException("could not make markings from: " + fv.getVisibility()); + throw new RuntimeException("could not make markings from: " + fvExpression); } } } - ColumnVisibility columnVisibility = null; + AccessExpression combinedExpression = null; try { - columnVisibility = this.markingFunctions.combine(visibilitiesToMerge); - event.setMarkings(this.markingFunctions.translateFromColumnVisibility(columnVisibility)); + combinedExpression = this.markingFunctions.combine(expressionsToMerge); + event.setMarkings(this.markingFunctions.fromAccessExpression(combinedExpression)); } catch (Exception e1) { - throw new RuntimeException("could not make markings from: " + columnVisibility); + throw new RuntimeException("could not make markings from: " + combinedExpression); } event.setFields(new ArrayList<>(values)); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformerSupport.java b/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformerSupport.java index edf372e7021..089a5f2f953 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformerSupport.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformerSupport.java @@ -18,10 +18,12 @@ import com.esotericsoftware.kryo.Kryo; import com.google.common.collect.Lists; +import datawave.core.common.util.AccessExpressionUtil; import datawave.core.query.cachedresults.CacheableLogic; import datawave.core.query.logic.BaseQueryLogic; import datawave.core.query.logic.BaseQueryLogicTransformer; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.microservice.query.Query; import datawave.microservice.query.QueryImpl.Parameter; import datawave.query.model.QueryModel; @@ -101,7 +103,7 @@ public CacheableQueryRow writeToCache(Object o) throws QueryException { @Override public Object readFromCache(CacheableQueryRow cacheableQueryRow) { - Map markings = cacheableQueryRow.getMarkings(); + Markings markings = cacheableQueryRow.getMarkings(); String dataType = cacheableQueryRow.getDataType(); String internalId = cacheableQueryRow.getEventId(); String row = cacheableQueryRow.getRow(); @@ -121,7 +123,7 @@ public Object readFromCache(CacheableQueryRow cacheableQueryRow) { for (Entry entry : columnValueMap.entrySet()) { String columnName = entry.getKey(); String columnValue = entry.getValue(); - Map columnMarkings = cacheableQueryRow.getColumnMarkings(columnName); + Markings columnMarkings = cacheableQueryRow.getColumnMarkings(columnName); String columnVisibility = cacheableQueryRow.getColumnVisibility(columnName); Long columnTimestamp = cacheableQueryRow.getColumnTimestamp(columnName); FieldBase field = this.makeField(columnName, columnMarkings, columnVisibility, columnTimestamp, columnValue); @@ -150,7 +152,7 @@ public BaseQueryResponse createResponse(List resultList) { return response; } - protected FieldBase makeField(String name, Map markings, String columnVisibility, Long timestamp, Object value) { + protected FieldBase makeField(String name, Markings markings, String columnVisibility, Long timestamp, Object value) { FieldBase field = this.responseObjectFactory.getField(); field.setName(name); field.setMarkings(markings); @@ -160,9 +162,9 @@ protected FieldBase makeField(String name, Map markings, Strin return field; } - protected FieldBase makeField(String name, Map markings, ColumnVisibility columnVisibility, Long timestamp, Object value) { + protected FieldBase makeField(String name, Markings markings, ColumnVisibility columnVisibility, Long timestamp, Object value) { FieldBase field = makeField(name, markings, (String) null, timestamp, value); - field.setColumnVisibility(columnVisibility); + field.setColumnVisibility(AccessExpressionUtil.toAccessExpression(columnVisibility)); return field; } diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java index 6fb9c45ddf2..01f05024134 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java @@ -8,6 +8,7 @@ import java.util.Map.Entry; import java.util.Set; +import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.ColumnVisibility; @@ -16,6 +17,7 @@ import com.google.common.base.Preconditions; +import datawave.core.common.util.AccessExpressionUtil; import datawave.core.query.exception.EmptyObjectException; import datawave.core.query.logic.BaseQueryLogic; import datawave.data.type.StringType; @@ -126,9 +128,11 @@ protected Collection buildFacets(Key documentKey, String f FieldCardinalityBase fc = this.responseObjectFactory.getFieldCardinality(); fc.setField(v.getFieldName()); - fc.setMarkings(markingFunctions.translateFromColumnVisibilityForAuths(attr.getColumnVisibility(), auths)); // reduces colvis based on - // visibility - fc.setColumnVisibility(new String(markingFunctions.translateToColumnVisibility(fc.getMarkings()).flatten())); + fc.setMarkings(markingFunctions.fromAccessExpression(AccessExpressionUtil.toAccessExpression(attr.getColumnVisibility()))); // reduces + // colvis based + // on + // visibility + fc.setColumnVisibility(MarkingFunctions.normalize(markingFunctions.toAccessExpression(fc.getMarkings())).getExpression()); fc.setLower(v.getFloorValue()); fc.setUpper(v.getCeilingValue()); fc.setCardinality(v.getEstimate().cardinality()); @@ -152,7 +156,7 @@ protected FacetsBase buildResponse(Document document, Key documentKey, ColumnVis final Collection documentFields = buildFacets(documentKey, null, document, eventCV, mf); - facetedResponse.setMarkings(mf.translateFromColumnVisibility(eventCV)); + facetedResponse.setMarkings(mf.fromAccessExpression(AccessExpressionUtil.toAccessExpression(eventCV))); facetedResponse.setFields(new ArrayList<>(documentFields)); // assign an estimate of the event size based on the document size @@ -166,7 +170,7 @@ protected FacetsBase buildResponse(Document document, Key documentKey, ColumnVis @Override public BaseQueryResponse createResponse(List resultList) { FacetQueryResponseBase response = responseObjectFactory.getFacetQueryResponse(); - Set combinedColumnVisibility = new HashSet<>(); + Set combinedExpressions = new HashSet<>(); for (Object result : resultList) { FacetsBase facet = (FacetsBase) result; @@ -177,14 +181,14 @@ public BaseQueryResponse createResponse(List resultList) { for (FieldCardinalityBase fcb : facet.getFields()) { if (StringUtils.isNotBlank(fcb.getColumnVisibility())) { - combinedColumnVisibility.add(new ColumnVisibility(fcb.getColumnVisibility())); + combinedExpressions.add(AccessExpressionUtil.toAccessExpression(new ColumnVisibility(fcb.getColumnVisibility()))); } } } try { - ColumnVisibility columnVisibility = this.markingFunctions.combine(combinedColumnVisibility); - response.setMarkings(this.markingFunctions.translateFromColumnVisibility(columnVisibility)); + AccessExpression combinedExpression = this.markingFunctions.combine(combinedExpressions); + response.setMarkings(this.markingFunctions.fromAccessExpression(combinedExpression)); } catch (MarkingFunctions.Exception e) { log.warn(e); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/FieldIndexCountQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/FieldIndexCountQueryTransformer.java index f570fa4fc15..456b1138630 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/FieldIndexCountQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/FieldIndexCountQueryTransformer.java @@ -11,11 +11,13 @@ import org.apache.hadoop.io.Text; import org.apache.log4j.Logger; +import datawave.core.common.util.AccessExpressionUtil; import datawave.core.query.cachedresults.CacheableLogic; import datawave.core.query.logic.BaseQueryLogic; import datawave.core.query.logic.BaseQueryLogicTransformer; import datawave.data.hash.UID; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.microservice.query.Query; import datawave.query.Constants; import datawave.query.tables.shard.FieldIndexCountQueryLogic.Tuple; @@ -59,9 +61,9 @@ public EventBase transform(Entry entry) { String key = entry.getKey(); Tuple val = entry.getValue(); - Map markings = null; + Markings markings = null; try { - markings = this.markingFunctions.translateFromColumnVisibilityForAuths(val.getColumnVisibility(), this.auths); + markings = this.markingFunctions.fromAccessExpression(AccessExpressionUtil.toAccessExpression(val.getColumnVisibility())); } catch (Exception e) { log.error("could not translate " + val.getColumnVisibility() + " to markings, skipping entry"); return null; @@ -171,7 +173,7 @@ public CacheableQueryRow writeToCache(Object o) throws QueryException { return cqo; } - private FieldBase makeField(String name, Map markings, String columnVisibility, Long timestamp, Object value) { + private FieldBase makeField(String name, Markings markings, String columnVisibility, Long timestamp, Object value) { FieldBase field = this.responseObjectFactory.getField(); field.setName(name); field.setMarkings(markings); @@ -186,7 +188,7 @@ public Object readFromCache(CacheableQueryRow cacheableQueryRow) { if (this.variableFieldList == null) { this.variableFieldList = cacheableQueryRow.getVariableColumnNames(); } - Map markings = cacheableQueryRow.getMarkings(); + Markings markings = cacheableQueryRow.getMarkings(); String dataType = cacheableQueryRow.getDataType(); String internalId = cacheableQueryRow.getEventId(); String row = cacheableQueryRow.getRow(); @@ -206,7 +208,7 @@ public Object readFromCache(CacheableQueryRow cacheableQueryRow) { for (Map.Entry entry : columnValueMap.entrySet()) { String columnName = entry.getKey(); String columnValue = entry.getValue(); - Map columnMarkings = cacheableQueryRow.getColumnMarkings(columnName); + Markings columnMarkings = cacheableQueryRow.getColumnMarkings(columnName); String columnVisibility = cacheableQueryRow.getColumnVisibility(columnName); Long columnTimestamp = cacheableQueryRow.getColumnTimestamp(columnName); FieldBase field = this.makeField(columnName, columnMarkings, columnVisibility, columnTimestamp, columnValue); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/GroupingDocumentTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/GroupingDocumentTransformer.java index 9c764087cdf..949f3408aaa 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/GroupingDocumentTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/GroupingDocumentTransformer.java @@ -21,8 +21,11 @@ import com.google.common.collect.Sets; import com.google.common.collect.TreeMultimap; +import datawave.core.common.util.AccessExpressionUtil; import datawave.core.query.logic.BaseQueryLogic; +import datawave.marking.AccessExpressionMarkings; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.microservice.query.Query; import datawave.query.model.QueryModel; import datawave.query.tables.ShardQueryLogic; @@ -86,7 +89,7 @@ public BaseQueryResponse createResponse(List resultList) { } protected BaseQueryResponse createGroupedResponse(Multiset>> multiset) { - Map markings = Maps.newHashMap(); + Markings markings = AccessExpressionMarkings.builder().build(); EventQueryResponseBase response = this.responseObjectFactory.getEventQueryResponse(); List events = new ArrayList<>(); for (Collection> entry : multiset.elementSet()) { @@ -128,7 +131,9 @@ private Multimap getFieldToFieldWithGroupingContextMap(Collection log.trace(this.groupFieldsList + " contains " + shorterName); FieldBase created = null; try { - created = this.makeField(shortName, this.markingFunctions.translateFromColumnVisibility(new ColumnVisibility(field.getColumnVisibility())), + created = this.makeField(shortName, + this.markingFunctions.fromAccessExpression( + AccessExpressionUtil.toAccessExpression(new ColumnVisibility(field.getColumnVisibility()))), field.getColumnVisibility(), 0L, field.getValueOfTypedValue()); } catch (Exception ex) { log.error(ex); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/ShardIndexQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/ShardIndexQueryTransformer.java index 8a1d6f8a60e..c754eb0a348 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/ShardIndexQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/ShardIndexQueryTransformer.java @@ -14,12 +14,14 @@ import com.google.protobuf.InvalidProtocolBufferException; +import datawave.core.common.util.AccessExpressionUtil; import datawave.core.query.cachedresults.CacheableLogic; import datawave.core.query.logic.BaseQueryLogic; import datawave.core.query.logic.BaseQueryLogicTransformer; import datawave.ingest.protobuf.Uid; import datawave.marking.MarkingFunctions; import datawave.marking.MarkingFunctions.Exception; +import datawave.marking.Markings; import datawave.microservice.query.Query; import datawave.query.model.QueryModel; import datawave.webservice.query.cachedresults.CacheableQueryRow; @@ -66,9 +68,9 @@ public EventBase transform(Entry input) { EventBase event = responseObjectFactory.getEvent(); ColumnVisibility columnVisibility = new ColumnVisibility(entry.getKey().getColumnVisibility()); - Map markings; + Markings markings; try { - markings = this.markingFunctions.translateFromColumnVisibilityForAuths(columnVisibility, this.auths); + markings = this.markingFunctions.fromAccessExpression(AccessExpressionUtil.toAccessExpression(columnVisibility)); } catch (Exception e1) { throw new RuntimeException("could not make markings from: " + columnVisibility); } @@ -113,7 +115,7 @@ public EventBase transform(Entry input) { return event; } - private FieldBase makeField(String name, Map markings, String columnVisibility, Long timestamp, Object value) { + private FieldBase makeField(String name, Markings markings, String columnVisibility, Long timestamp, Object value) { FieldBase field = this.responseObjectFactory.getField(); field.setName(name); field.setMarkings(markings); @@ -166,7 +168,7 @@ public Object readFromCache(CacheableQueryRow cacheableQueryRow) { if (this.variableFieldList == null) { this.variableFieldList = cacheableQueryRow.getVariableColumnNames(); } - Map markings = cacheableQueryRow.getMarkings(); + Markings markings = cacheableQueryRow.getMarkings(); String dataType = cacheableQueryRow.getDataType(); String internalId = cacheableQueryRow.getEventId(); String row = cacheableQueryRow.getRow(); @@ -189,7 +191,7 @@ public Object readFromCache(CacheableQueryRow cacheableQueryRow) { String columnValue = entry.getValue(); String columnVisibility = cacheableQueryRow.getColumnVisibility(columnName); Long columnTimestamp = cacheableQueryRow.getColumnTimestamp(columnName); - Map columnMarkings = cacheableQueryRow.getColumnMarkings(columnName); + Markings columnMarkings = cacheableQueryRow.getColumnMarkings(columnName); FieldBase field = responseObjectFactory.getField(); field.setName(columnName); field.setMarkings(columnMarkings); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/ShardQueryCountTableTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/ShardQueryCountTableTransformer.java index 95b852c455b..fad81c8f7d5 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/ShardQueryCountTableTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/ShardQueryCountTableTransformer.java @@ -9,10 +9,12 @@ import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.log4j.Logger; +import datawave.core.common.util.AccessExpressionUtil; import datawave.core.query.cachedresults.CacheableLogic; import datawave.core.query.logic.BaseQueryLogicTransformer; import datawave.marking.MarkingFunctions; import datawave.marking.MarkingFunctions.Exception; +import datawave.marking.Markings; import datawave.microservice.query.Query; import datawave.query.Constants; import datawave.webservice.query.cachedresults.CacheableQueryRow; @@ -46,9 +48,9 @@ public EventBase transform(Entry untypedEntry) { Long count = untypedEntry.getKey(); ColumnVisibility vis = untypedEntry.getValue(); - Map markings; + Markings markings; try { - markings = markingFunctions.translateFromColumnVisibilityForAuths(vis, auths); + markings = markingFunctions.fromAccessExpression(AccessExpressionUtil.toAccessExpression(vis)); } catch (Exception e1) { throw new IllegalArgumentException("Unable to translate markings", e1); } @@ -72,11 +74,11 @@ public EventBase transform(Entry untypedEntry) { return e; } - private FieldBase makeField(String name, Map markings, ColumnVisibility columnVisibility, Long timestamp, Object value) { + private FieldBase makeField(String name, Markings markings, ColumnVisibility columnVisibility, Long timestamp, Object value) { FieldBase field = this.responseObjectFactory.getField(); field.setName(name); field.setMarkings(markings); - field.setColumnVisibility(columnVisibility); + field.setColumnVisibility(AccessExpressionUtil.toAccessExpression(columnVisibility)); field.setTimestamp(timestamp); field.setValue(value); return field; @@ -125,7 +127,7 @@ public Object readFromCache(CacheableQueryRow cacheableQueryRow) { if (this.variableFieldList == null) { this.variableFieldList = cacheableQueryRow.getVariableColumnNames(); } - Map markings = cacheableQueryRow.getMarkings(); + Markings markings = cacheableQueryRow.getMarkings(); String dataType = cacheableQueryRow.getDataType(); String internalId = cacheableQueryRow.getEventId(); String row = cacheableQueryRow.getRow(); @@ -147,7 +149,7 @@ public Object readFromCache(CacheableQueryRow cacheableQueryRow) { String columnValue = entry.getValue(); String columnVisibility = cacheableQueryRow.getColumnVisibility(columnName); Long columnTimestamp = cacheableQueryRow.getColumnTimestamp(columnName); - Map columnMarkings = cacheableQueryRow.getColumnMarkings(columnName); + Markings columnMarkings = cacheableQueryRow.getColumnMarkings(columnName); FieldBase field = responseObjectFactory.getField(); field.setName(columnName); field.setMarkings(columnMarkings); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/TagCloudTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/TagCloudTransformer.java index a1b3431a7a5..e4847f1dc11 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/TagCloudTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/TagCloudTransformer.java @@ -2,6 +2,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.stream.Collectors; @@ -9,6 +10,8 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.Authorizations; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import datawave.core.query.logic.BaseQueryLogicTransformer; import datawave.marking.MarkingFunctions; @@ -35,6 +38,8 @@ public class TagCloudTransformer extends BaseQueryLogicTransformer> transformers; + private static final Logger log = LoggerFactory.getLogger(TagCloudTransformer.class); + public enum ResponseVersion { V1, V2 } @@ -183,9 +188,7 @@ private void configureV1TagCloud(TagCloudBase base, TagCloud tagCloud) { } } } - if (!tagCloud.getVisibility().isEmpty()) { - base.setMarkings(tagCloud.getVisibility()); - } + convertAndSetMarkings(base, tagCloud.getVisibility()); base.setTags(generateTagCloudEntries(tagCloud)); } @@ -193,12 +196,24 @@ private void configureV2TagCloud(TagCloudBase base, TagCloud tagCloud) { if (tagCloud.getMetadata() != null && !tagCloud.getMetadata().isEmpty()) { base.setMetadata(tagCloud.getMetadata()); } - if (!tagCloud.getVisibility().isEmpty()) { - base.setMarkings(tagCloud.getVisibility()); - } + convertAndSetMarkings(base, tagCloud.getVisibility()); base.setTags(generateTagCloudEntries(tagCloud)); } + @SuppressWarnings("unchecked") + private void convertAndSetMarkings(TagCloudBase base, Map visibility) { + if (visibility != null && !visibility.isEmpty()) { + String expr = visibility.getOrDefault("visibility", ""); + if (!expr.isEmpty()) { + try { + base.setMarkings(markingFunctions.fromAccessExpression(MarkingFunctions.Default.ACCESS.newExpression(expr))); + } catch (MarkingFunctions.Exception e) { + log.warn("Could not convert tag cloud visibility to markings: {}", expr, e); + } + } + } + } + private void configureTagCloud(TagCloudBase base, TagCloud tagCloud) { if (responseVersion == ResponseVersion.V1) { configureV1TagCloud(base, tagCloud); diff --git a/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleEvent.java b/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleEvent.java index fa2b205ef8e..c87c269e7a0 100644 --- a/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleEvent.java +++ b/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleEvent.java @@ -5,7 +5,6 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; -import java.util.Map; import javax.xml.bind.annotation.XmlAccessOrder; import javax.xml.bind.annotation.XmlAccessType; @@ -15,8 +14,6 @@ import javax.xml.bind.annotation.XmlElementWrapper; import javax.xml.bind.annotation.XmlTransient; -import com.google.common.collect.Maps; - import datawave.webservice.query.data.ObjectSizeOf; import io.protostuff.Input; import io.protostuff.Message; @@ -29,9 +26,6 @@ public class SimpleEvent extends EventBase implements S private static final long serialVersionUID = -1963967131747306439L; - @XmlElementWrapper(name = "Markings") - private Map markings; - @XmlElement(name = "Metadata") private Metadata metadata = null; @@ -102,9 +96,7 @@ public void writeTo(Output output, SimpleEvent message) throws IOException { output.writeObject(2, field, SimpleField.getSchema(), true); } } - if (message.markings != null) { - output.writeObject(3, message.markings, MapSchema.SCHEMA, false); - } + // markings field 3 no longer written; markings are now on the parent EventBase } public void mergeFrom(Input input, SimpleEvent message) throws IOException { @@ -121,9 +113,9 @@ public void mergeFrom(Input input, SimpleEvent message) throws IOException { message.fields.add(f); break; case 3: - if (message.markings == null) - message.markings = Maps.newHashMap(); - input.mergeObject(message.markings, MapSchema.SCHEMA); + // legacy markings map — consume but discard + HashMap legacyMarkings = new HashMap<>(); + input.mergeObject(legacyMarkings, MapSchema.SCHEMA); default: input.handleUnknownField(number, this); break; diff --git a/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleField.java b/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleField.java index 8363d8805cf..5994c3aadde 100644 --- a/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleField.java +++ b/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleField.java @@ -17,9 +17,10 @@ import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; -import com.google.common.collect.Maps; - import datawave.data.type.Type; +import datawave.marking.AccessExpressionMarkings; +import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.webservice.query.util.TypedValue; import io.protostuff.Input; import io.protostuff.Message; @@ -38,7 +39,7 @@ public class SimpleField extends FieldBase implements Serializable, @XmlAttribute(name = "columnVisibility") private String columnVisibility; @XmlElementWrapper(name = "Markings") - private Map markings; + private Markings markings; @XmlAttribute(name = "timestamp") private Long timestamp; @XmlAttribute(name = "name") @@ -48,7 +49,7 @@ public class SimpleField extends FieldBase implements Serializable, public SimpleField() {} - public SimpleField(String name, Map markings, String columnVisibility, Long timestamp, Object value) { + public SimpleField(String name, Markings markings, String columnVisibility, Long timestamp, Object value) { super(); this.name = name; this.markings = markings; @@ -135,16 +136,16 @@ public void setName(String name) { } private void assureMarkings() { - if (this.markings == null) - this.markings = Maps.newHashMap(); + // no-op: markings are now Markings and should be set externally } - public Map getMarkings() { - this.assureMarkings(); - return markings; + @Override + public Markings getMarkings() { + return this.markings; } - public void setMarkings(Map markings) { + @Override + public void setMarkings(Markings markings) { this.markings = markings; } @@ -198,7 +199,11 @@ public void writeTo(Output output, SimpleField message) throws IOException { if (message.value != null) output.writeObject(4, message.value, message.value.cachedSchema(), false); if (message.markings != null) { - output.writeObject(5, message.markings, MAP_SCHEMA, false); + // Serialize markings as the expression string for backward compat + if (message.markings instanceof AccessExpressionMarkings) { + String expr = ((AccessExpressionMarkings) message.markings).getAccessExpression().getExpression(); + output.writeString(5, expr, false); + } } } @@ -220,10 +225,10 @@ public void mergeFrom(Input input, SimpleField message) throws IOException { message.value = input.mergeObject(null, TypedValue.getSchema()); break; case 5: - if (message.markings == null) - message.markings = Maps.newHashMap(); - Map markings = input.mergeObject(null, MAP_SCHEMA); - message.markings.putAll(markings); + String expr = input.readString(); + if (expr != null && !expr.isEmpty()) { + message.markings = AccessExpressionMarkings.builder().accessExpression(MarkingFunctions.Default.ACCESS.newExpression(expr)).build(); + } default: input.handleUnknownField(number, this); break; diff --git a/warehouse/query-core/src/test/java/datawave/query/common/grouping/DocumentGrouperTest.java b/warehouse/query-core/src/test/java/datawave/query/common/grouping/DocumentGrouperTest.java index f695791d713..86d78e91b0e 100644 --- a/warehouse/query-core/src/test/java/datawave/query/common/grouping/DocumentGrouperTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/common/grouping/DocumentGrouperTest.java @@ -9,6 +9,8 @@ import java.util.Map; import java.util.Set; +import org.apache.accumulo.access.Access; +import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.security.ColumnVisibility; import org.assertj.core.util.Sets; @@ -37,6 +39,12 @@ public class DocumentGrouperTest { private static final ColumnVisibility COLVIS_E = new ColumnVisibility("E"); private static final ColumnVisibility COLVIS_I = new ColumnVisibility("I"); private static final ColumnVisibility COLVIS_ALL_E_I = new ColumnVisibility("ALL&E&I"); + + private static final Access ACCESS = Access.builder().build(); + private static final AccessExpression AE_ALL = ACCESS.newExpression("ALL"); + private static final AccessExpression AE_E = ACCESS.newExpression("E"); + private static final AccessExpression AE_I = ACCESS.newExpression("I"); + private static final AccessExpression AE_ALL_E_I = ACCESS.newExpression("ALL&E&I"); private static final Key key = new Key("test_key"); private static final Multimap inverseReverseMap = HashMultimap.create(); private static final Map reverseMap = new HashMap<>(); @@ -526,10 +534,10 @@ public void testGroupingBySingleFieldAcrossMultipleDocuments() { GroupsAssert groupsAssert = GroupsAssert.assertThat(groups); groupsAssert.hasTotalGroups(2); - groupsAssert.assertGroup(textKey("GENDER", "MALE")).hasCount(4).hasDocumentVisibilities(COLVIS_ALL, COLVIS_I) - .hasVisibilitiesForKey(textKey("GENDER", "MALE"), COLVIS_ALL, COLVIS_I); - groupsAssert.assertGroup(textKey("GENDER", "FEMALE")).hasCount(3).hasDocumentVisibilities(COLVIS_I, COLVIS_E) - .hasVisibilitiesForKey(textKey("GENDER", "FEMALE"), COLVIS_I, COLVIS_E); + groupsAssert.assertGroup(textKey("GENDER", "MALE")).hasCount(4).hasDocumentExpressions(AE_ALL, AE_I) + .hasAccessExpressionsForKey(textKey("GENDER", "MALE"), AE_ALL, AE_I); + groupsAssert.assertGroup(textKey("GENDER", "FEMALE")).hasCount(3).hasDocumentExpressions(AE_I, AE_E) + .hasAccessExpressionsForKey(textKey("GENDER", "FEMALE"), AE_I, AE_E); } @Test @@ -613,18 +621,18 @@ public void testGroupingByMultipleFieldsWithDifferentFormatsAcrossMultipleDocume GroupsAssert groupsAssert = GroupsAssert.assertThat(groups); groupsAssert.hasTotalGroups(7); - groupsAssert.assertGroup(textKey("GENDER", "MALE"), textKey("BUILDING", "West")).hasCount(1).hasDocumentVisibilities(COLVIS_ALL_E_I) - .hasVisibilitiesForKey(textKey("GENDER", "MALE"), COLVIS_ALL).hasVisibilitiesForKey(textKey("BUILDING", "West"), COLVIS_E); - groupsAssert.assertGroup(textKey("GENDER", "MALE"), textKey("BUILDING", "East")).hasCount(1).hasDocumentVisibilities(COLVIS_ALL_E_I) - .hasVisibilitiesForKey(textKey("GENDER", "MALE"), COLVIS_ALL).hasVisibilitiesForKey(textKey("BUILDING", "East"), COLVIS_I); - groupsAssert.assertGroup(textKey("GENDER", "FEMALE"), textKey("BUILDING", "West")).hasCount(1).hasDocumentVisibilities(COLVIS_ALL_E_I) - .hasVisibilitiesForKey(textKey("GENDER", "FEMALE"), COLVIS_ALL).hasVisibilitiesForKey(textKey("BUILDING", "West"), COLVIS_E); - groupsAssert.assertGroup(textKey("GENDER", "FEMALE"), textKey("BUILDING", "East")).hasCount(1).hasDocumentVisibilities(COLVIS_ALL_E_I) - .hasVisibilitiesForKey(textKey("GENDER", "FEMALE"), COLVIS_ALL).hasVisibilitiesForKey(textKey("BUILDING", "East"), COLVIS_I); - groupsAssert.assertGroup(textKey("GENDER", "MALE"), textKey("BUILDING", "Center")).hasCount(2).hasDocumentVisibilities(COLVIS_ALL) - .hasVisibilitiesForKey(textKey("GENDER", "MALE"), COLVIS_ALL).hasVisibilitiesForKey(textKey("BUILDING", "Center"), COLVIS_ALL); - groupsAssert.assertGroup(textKey("BUILDING", "North")).hasCount(1).hasDocumentVisibilities(COLVIS_ALL); - groupsAssert.assertGroup(textKey("BUILDING", "South")).hasCount(1).hasDocumentVisibilities(COLVIS_ALL); + groupsAssert.assertGroup(textKey("GENDER", "MALE"), textKey("BUILDING", "West")).hasCount(1).hasDocumentExpressions(AE_ALL_E_I) + .hasAccessExpressionsForKey(textKey("GENDER", "MALE"), AE_ALL).hasAccessExpressionsForKey(textKey("BUILDING", "West"), AE_E); + groupsAssert.assertGroup(textKey("GENDER", "MALE"), textKey("BUILDING", "East")).hasCount(1).hasDocumentExpressions(AE_ALL_E_I) + .hasAccessExpressionsForKey(textKey("GENDER", "MALE"), AE_ALL).hasAccessExpressionsForKey(textKey("BUILDING", "East"), AE_I); + groupsAssert.assertGroup(textKey("GENDER", "FEMALE"), textKey("BUILDING", "West")).hasCount(1).hasDocumentExpressions(AE_ALL_E_I) + .hasAccessExpressionsForKey(textKey("GENDER", "FEMALE"), AE_ALL).hasAccessExpressionsForKey(textKey("BUILDING", "West"), AE_E); + groupsAssert.assertGroup(textKey("GENDER", "FEMALE"), textKey("BUILDING", "East")).hasCount(1).hasDocumentExpressions(AE_ALL_E_I) + .hasAccessExpressionsForKey(textKey("GENDER", "FEMALE"), AE_ALL).hasAccessExpressionsForKey(textKey("BUILDING", "East"), AE_I); + groupsAssert.assertGroup(textKey("GENDER", "MALE"), textKey("BUILDING", "Center")).hasCount(2).hasDocumentExpressions(AE_ALL) + .hasAccessExpressionsForKey(textKey("GENDER", "MALE"), AE_ALL).hasAccessExpressionsForKey(textKey("BUILDING", "Center"), AE_ALL); + groupsAssert.assertGroup(textKey("BUILDING", "North")).hasCount(1).hasDocumentExpressions(AE_ALL); + groupsAssert.assertGroup(textKey("BUILDING", "South")).hasCount(1).hasDocumentExpressions(AE_ALL); } @Test diff --git a/warehouse/query-core/src/test/java/datawave/query/index/day/TruncatedIndexIngest.java b/warehouse/query-core/src/test/java/datawave/query/index/day/TruncatedIndexIngest.java index 3cddd47476d..89a2174ed39 100644 --- a/warehouse/query-core/src/test/java/datawave/query/index/day/TruncatedIndexIngest.java +++ b/warehouse/query-core/src/test/java/datawave/query/index/day/TruncatedIndexIngest.java @@ -26,7 +26,6 @@ import datawave.ingest.table.aggregator.BitSetCombiner; import datawave.ingest.table.aggregator.util.TruncatedIndexKeyParser; -import datawave.marking.ColumnVisibilityCache; /** * Utility that converts a standard shard index to a truncated index @@ -86,7 +85,7 @@ public void convert(AccumuloClient client, Authorizations auths, String sourceTa for (Map.Entry entry : combined.entries()) { Key k = entry.getKey(); Mutation m = new Mutation(k.getRow()); - ColumnVisibility cv = ColumnVisibilityCache.get(k.getColumnVisibilityData()); + ColumnVisibility cv = k.getColumnVisibilityParsed(); m.put(k.getColumnFamily().toString(), k.getColumnQualifier().toString(), cv, k.getTimestamp(), entry.getValue()); bw.addMutation(m); } diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/RemoteEventQueryLogicTest.java b/warehouse/query-core/src/test/java/datawave/query/tables/RemoteEventQueryLogicTest.java index 3a07901af58..2a9cf9cc9b9 100644 --- a/warehouse/query-core/src/test/java/datawave/query/tables/RemoteEventQueryLogicTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/tables/RemoteEventQueryLogicTest.java @@ -6,7 +6,6 @@ import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; import java.util.List; @@ -48,13 +47,13 @@ public void setup() { DefaultEventQueryResponse response1 = new DefaultEventQueryResponse(); DefaultEvent event1 = new DefaultEvent(); - event1.setFields(Collections.singletonList(new DefaultField("FOO1", "FOO|BAR", new HashMap(), -1L, "FOOBAR1"))); + event1.setFields(Collections.singletonList(new DefaultField("FOO1", "FOO|BAR", -1L, "FOOBAR1"))); response1.setEvents(Collections.singletonList(event1)); response1.setReturnedEvents(1L); DefaultEventQueryResponse response2 = new DefaultEventQueryResponse(); DefaultEvent event2 = new DefaultEvent(); - event1.setFields(Collections.singletonList(new DefaultField("FOO2", "FOO|BAR", new HashMap(), -1L, "FOOBAR2"))); + event1.setFields(Collections.singletonList(new DefaultField("FOO2", "FOO|BAR", -1L, "FOOBAR2"))); response2.setEvents(Collections.singletonList(event1)); response2.setReturnedEvents(1L); diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/RemoteQueryServiceTestUtil.java b/warehouse/query-core/src/test/java/datawave/query/tables/RemoteQueryServiceTestUtil.java index 5c7ad67954f..fd5763ebb41 100644 --- a/warehouse/query-core/src/test/java/datawave/query/tables/RemoteQueryServiceTestUtil.java +++ b/warehouse/query-core/src/test/java/datawave/query/tables/RemoteQueryServiceTestUtil.java @@ -23,7 +23,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Date; -import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.UUID; @@ -155,7 +154,7 @@ public void initialize() throws IOException { DefaultEventQueryResponse response = new DefaultEventQueryResponse(); if (nextCount < totalNext) { DefaultEvent event = new DefaultEvent(); - event.setFields(Collections.singletonList(new DefaultField("FOO" + nextCount, "FOO|BAR", new HashMap(), -1L, "FOOBAR" + nextCount))); + event.setFields(Collections.singletonList(new DefaultField("FOO" + nextCount, "FOO|BAR", -1L, "FOOBAR" + nextCount))); response.setEvents(Collections.singletonList(event)); response.setReturnedEvents(1L); } else { diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/content/ContentQueryLogicTest.java b/warehouse/query-core/src/test/java/datawave/query/tables/content/ContentQueryLogicTest.java index 6d83a2af5dd..f86be83688b 100644 --- a/warehouse/query-core/src/test/java/datawave/query/tables/content/ContentQueryLogicTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/tables/content/ContentQueryLogicTest.java @@ -55,7 +55,7 @@ public class ContentQueryLogicTest { private ContentQueryConfiguration mockContentConfig; private Query query; - private final MarkingFunctions markingFunctions = MarkingFunctionsFactory.createMarkingFunctions(); + private final MarkingFunctions markingFunctions = MarkingFunctionsFactory.createMarkingFunctions(); @BeforeEach public void setup() throws TableNotFoundException { @@ -171,7 +171,7 @@ public void testDecodeViewParam() throws Exception { Value viewValue = new Value(Base64.encodeBytes("my happy message".getBytes())); Map.Entry entry = new AbstractMap.SimpleImmutableEntry<>(dataKey, viewValue); - markingFunctions.translateFromColumnVisibilityForAuths(new ColumnVisibility("A"), auths); + markingFunctions.fromAccessExpression(new ColumnVisibility("A"), auths); // test with decode view logic.initialize(mockClient, settings, Set.of(auths)); diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/keyword/TagCloudTestUtil.java b/warehouse/query-core/src/test/java/datawave/query/tables/keyword/TagCloudTestUtil.java index c10197377fc..919d77eceab 100644 --- a/warehouse/query-core/src/test/java/datawave/query/tables/keyword/TagCloudTestUtil.java +++ b/warehouse/query-core/src/test/java/datawave/query/tables/keyword/TagCloudTestUtil.java @@ -4,6 +4,10 @@ import java.util.List; import java.util.Map; +import org.apache.accumulo.access.Access; + +import datawave.marking.AccessExpressionMarkings; +import datawave.marking.Markings; import datawave.query.tables.keyword.transform.KeywordResultsTransformer; import datawave.webservice.result.keyword.DefaultTagCloud; import datawave.webservice.result.keyword.DefaultTagCloudEntry; @@ -12,6 +16,9 @@ * Utility for producing TagClouds and supporting objects for tests */ public class TagCloudTestUtil { + private static final Access ACCESS = Access.builder().build(); + private static final Markings ALL_MARKINGS = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression("ALL")).build(); + public static final String SOPRANO_SOURCE = "20130101_0/test/-1kfeoq.-80b5fs.r0262j"; public static final String CAPONE_SOURCE = "20130101_0/test/-cvy0gj.tlf59s.-duxzua"; public static final String CORLEONE_SOURCE = "20130101_0/test/-d5uxna.msizfm.-oxy0iu"; @@ -27,7 +34,7 @@ public DefaultTagCloud getExpectedCloud(String version, Map metad } else { expectedCloud.setMetadata(metadata); } - expectedCloud.setMarkings(Map.of("visibility", "[ALL]")); + expectedCloud.setMarkings(ALL_MARKINGS); expectedCloud.setTags(entries); expectedCloud.setIntermediateResult(false); @@ -45,7 +52,7 @@ public DefaultTagCloud getCaponeKeywordCloud(String docId, String version) { entries.add(createTagCloudEntry("much farther", 0.5903, 1, List.of(docId))); DefaultTagCloud expectedCloud = new DefaultTagCloud(); - expectedCloud.setMarkings(Map.of("visibility", "[ALL]")); + expectedCloud.setMarkings(ALL_MARKINGS); expectedCloud.setTags(entries); if (version.equals("1")) { expectedCloud.setLanguage("keyword"); diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/testframework/SSDeepDataType.java b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/testframework/SSDeepDataType.java index 2acb83e6943..4b935e6349e 100644 --- a/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/testframework/SSDeepDataType.java +++ b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/testframework/SSDeepDataType.java @@ -21,7 +21,6 @@ import datawave.ingest.input.reader.EventRecordReader; import datawave.ingest.mapreduce.handler.shard.AbstractColumnBasedHandler; import datawave.ingest.mapreduce.handler.ssdeep.SSDeepIndexHandler; -import datawave.marking.MarkingFunctions; import datawave.query.testframework.AbstractDataTypeConfig; import datawave.query.testframework.FieldConfig; import datawave.query.testframework.RawMetaData; @@ -247,7 +246,7 @@ public String getSecurityMarkingFieldNames() { @Override public String getSecurityMarkingFieldDomains() { - return MarkingFunctions.Default.COLUMN_VISIBILITY; + return "columnVisibility"; } @Override diff --git a/warehouse/query-core/src/test/java/datawave/query/testframework/CitiesDataType.java b/warehouse/query-core/src/test/java/datawave/query/testframework/CitiesDataType.java index 6111b298b0e..2d9efd43309 100644 --- a/warehouse/query-core/src/test/java/datawave/query/testframework/CitiesDataType.java +++ b/warehouse/query-core/src/test/java/datawave/query/testframework/CitiesDataType.java @@ -22,7 +22,6 @@ import datawave.ingest.data.config.CSVHelper; import datawave.ingest.data.config.ingest.BaseIngestHelper; import datawave.ingest.input.reader.EventRecordReader; -import datawave.marking.MarkingFunctions; /** * Contains all of the relevant data needed to configure any of the cities data types. @@ -256,7 +255,7 @@ public String getSecurityMarkingFieldNames() { @Override public String getSecurityMarkingFieldDomains() { - return MarkingFunctions.Default.COLUMN_VISIBILITY; + return "columnVisibility"; } @Override diff --git a/warehouse/query-core/src/test/java/datawave/query/testframework/cardata/CarsDataType.java b/warehouse/query-core/src/test/java/datawave/query/testframework/cardata/CarsDataType.java index c7adedc3016..9a66a903f21 100644 --- a/warehouse/query-core/src/test/java/datawave/query/testframework/cardata/CarsDataType.java +++ b/warehouse/query-core/src/test/java/datawave/query/testframework/cardata/CarsDataType.java @@ -20,7 +20,6 @@ import datawave.ingest.data.config.CSVHelper; import datawave.ingest.data.config.ingest.BaseIngestHelper; import datawave.ingest.input.reader.EventRecordReader; -import datawave.marking.MarkingFunctions; import datawave.query.testframework.AbstractDataTypeConfig; import datawave.query.testframework.FieldConfig; import datawave.query.testframework.RawMetaData; @@ -130,7 +129,7 @@ public static List eventSecurityFieldNames() { } public static List eventSecurityFieldDomains() { - return Collections.singletonList(MarkingFunctions.Default.COLUMN_VISIBILITY); + return Collections.singletonList("columnVisibility"); } /** diff --git a/warehouse/query-core/src/test/java/datawave/query/transformer/DocumentTransformerTest.java b/warehouse/query-core/src/test/java/datawave/query/transformer/DocumentTransformerTest.java index b80b565ef0f..ebc7f644bf1 100644 --- a/warehouse/query-core/src/test/java/datawave/query/transformer/DocumentTransformerTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/transformer/DocumentTransformerTest.java @@ -32,7 +32,7 @@ public class DocumentTransformerTest { // variables used to build the transformer private final Query query = new QueryImpl(); - private final MarkingFunctions markingFunctions = MarkingFunctionsFactory.createMarkingFunctions(); + private final MarkingFunctions markingFunctions = MarkingFunctionsFactory.createMarkingFunctions(); private final ResponseObjectFactory responseObjectFactory = new DefaultResponseObjectFactory(); private final KryoDocumentSerializer serializer = new KryoDocumentSerializer(); diff --git a/warehouse/query-core/src/test/java/datawave/query/transformer/GroupingTest.java b/warehouse/query-core/src/test/java/datawave/query/transformer/GroupingTest.java index 75fe059fb01..3b12ef9259e 100644 --- a/warehouse/query-core/src/test/java/datawave/query/transformer/GroupingTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/transformer/GroupingTest.java @@ -44,6 +44,7 @@ import datawave.core.query.iterator.DatawaveTransformIterator; import datawave.helpers.PrintUtility; import datawave.ingest.data.TypeRegistry; +import datawave.marking.Markings; import datawave.query.QueryParameters; import datawave.query.QueryTestTableHelper; import datawave.query.RebuildingScannerTestHelper; @@ -604,13 +605,17 @@ public void testGroupByWithReducedResponse() throws Exception { for (QueryResult result : queryResults) { // noinspection rawtypes for (EventBase event : result.response.getEvents()) { - String eventCV = event.getMarkings().get(COLVIS_MARKING).toString(); + Markings eventMarkings = event.getMarkings(); + Object eventMarkingsValue = eventMarkings != null ? eventMarkings.getMarkings() : null; + String eventCV = eventMarkingsValue != null ? eventMarkingsValue.toString() : null; assertThat(eventCV).describedAs("Assert event cv for teardown: %s, interrupt: %s", result.teardown, result.interrupt).isEqualTo(REDUCED_COLVIS); // noinspection unchecked for (FieldBase field : (List>) event.getFields()) { - String fieldCV = field.getMarkings().get(COLVIS_MARKING); - assertThat(fieldCV).describedAs("Assert null field cv for field: %s, teardown: %s, interrupt: %s", field.getName(), result.teardown, - result.interrupt).isNull(); + Markings fieldMarkings = field.getMarkings(); + assertThat(fieldMarkings == null || fieldMarkings.isEmpty()) + .describedAs("Assert empty field markings for field: %s, teardown: %s, interrupt: %s", field.getName(), result.teardown, + result.interrupt) + .isTrue(); } } } diff --git a/warehouse/query-core/src/test/java/datawave/query/util/NoUidIndexIngest.java b/warehouse/query-core/src/test/java/datawave/query/util/NoUidIndexIngest.java index ff7641b27d0..a55e1947006 100644 --- a/warehouse/query-core/src/test/java/datawave/query/util/NoUidIndexIngest.java +++ b/warehouse/query-core/src/test/java/datawave/query/util/NoUidIndexIngest.java @@ -27,7 +27,6 @@ import datawave.ingest.protobuf.Uid; import datawave.ingest.table.aggregator.KeepCountOnlyNoUidAggregator; -import datawave.marking.ColumnVisibilityCache; import datawave.query.index.day.AbstractIndexIngest; /** @@ -80,7 +79,7 @@ public void convert(AccumuloClient client, Authorizations auths, String source, for (Map.Entry entry : combined.entries()) { Key key = entry.getKey(); Mutation m = new Mutation(key.getRow()); - ColumnVisibility cv = ColumnVisibilityCache.get(key.getColumnVisibilityData()); + ColumnVisibility cv = key.getColumnVisibilityParsed(); m.put(key.getColumnFamily(), key.getColumnQualifier(), cv, key.getTimestamp(), entry.getValue()); bw.addMutation(m); } diff --git a/warehouse/query-core/src/test/java/datawave/test/GroupAssert.java b/warehouse/query-core/src/test/java/datawave/test/GroupAssert.java index 23c4c7c2c29..c8dc6e3d0c8 100644 --- a/warehouse/query-core/src/test/java/datawave/test/GroupAssert.java +++ b/warehouse/query-core/src/test/java/datawave/test/GroupAssert.java @@ -5,7 +5,7 @@ import java.util.Collection; import java.util.Objects; -import org.apache.accumulo.core.security.ColumnVisibility; +import org.apache.accumulo.access.AccessExpression; import org.assertj.core.api.AbstractAssert; import org.assertj.core.api.Assertions; @@ -34,20 +34,20 @@ public GroupAssert hasCount(int count) { return this; } - public GroupAssert hasVisibilitiesForKey(GroupingAttribute key, ColumnVisibility... visibilities) { + public GroupAssert hasAccessExpressionsForKey(GroupingAttribute key, AccessExpression... expressions) { isNotNull(); - Collection actualVisibilities = actual.getVisibilitiesForAttribute(key); - Assertions.assertThat(actualVisibilities).isNotNull().withFailMessage("Expected column visibilties for %s to contain exactly %s but was %s", key, - Arrays.toString(visibilities), actualVisibilities).containsExactlyInAnyOrder(visibilities); + Collection actualExpressions = actual.getAccessExpressionsForAttribute(key); + Assertions.assertThat(actualExpressions).isNotNull().withFailMessage("Expected access expressions for %s to contain exactly %s but was %s", key, + Arrays.toString(expressions), actualExpressions).containsExactlyInAnyOrder(expressions); return this; } - public GroupAssert hasDocumentVisibilities(ColumnVisibility... visibilities) { + public GroupAssert hasDocumentExpressions(AccessExpression... expressions) { isNotNull(); - Collection actualVisibilities = actual.getDocumentVisibilities(); - Assertions.assertThat(actualVisibilities).isNotNull() - .withFailMessage("Expected document visibilities to contain exactly %s but was %s", Arrays.toString(visibilities), actualVisibilities) - .containsExactlyInAnyOrder(visibilities); + Collection actualExpressions = actual.getDocumentExpressions(); + Assertions.assertThat(actualExpressions).isNotNull() + .withFailMessage("Expected document expressions to contain exactly %s but was %s", Arrays.toString(expressions), actualExpressions) + .containsExactlyInAnyOrder(expressions); return this; } diff --git a/web-services/cached-results/src/main/java/datawave/webservice/results/cached/CachedResultsBean.java b/web-services/cached-results/src/main/java/datawave/webservice/results/cached/CachedResultsBean.java index f416641273e..1c36a1e8180 100644 --- a/web-services/cached-results/src/main/java/datawave/webservice/results/cached/CachedResultsBean.java +++ b/web-services/cached-results/src/main/java/datawave/webservice/results/cached/CachedResultsBean.java @@ -26,7 +26,6 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; -import java.util.TreeMap; import java.util.UUID; import java.util.concurrent.Future; @@ -63,6 +62,7 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; +import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.commons.collections4.Transformer; import org.apache.commons.dbutils.DbUtils; @@ -94,7 +94,6 @@ import datawave.core.query.predict.QueryPredictor; import datawave.interceptor.RequiredInterceptor; import datawave.interceptor.ResponseInterceptor; -import datawave.marking.MarkingFunctions; import datawave.marking.SecurityMarking; import datawave.microservice.query.Query; import datawave.microservice.query.QueryParameters; @@ -303,7 +302,8 @@ protected void loadBatch(PreparedStatement ps, String owner, String queryId, Str ps.setString(5, cqo.getEventId()); ps.setString(6, cqo.getRow()); ps.setString(7, cqo.getColFam()); - ps.setString(8, MarkingFunctions.Encoding.toString(new TreeMap<>(cqo.getMarkings()))); + Object rawMarkings = (cqo.getMarkings() != null) ? cqo.getMarkings().getMarkings() : null; + ps.setString(8, (rawMarkings instanceof AccessExpression) ? ((AccessExpression) rawMarkings).getExpression() : ""); for (Entry e : cqo.getColumnValues().entrySet()) { String columnName = e.getKey(); @@ -475,7 +475,7 @@ protected GenericResponse load(@Required("queryId") String queryId, Stri try { MultiValueMap queryMap = new LinkedMultiValueMap<>(q.toMap()); marking.validate(queryMap); - queryMap.set(PrivateAuditConstants.COLUMN_VISIBILITY, marking.toColumnVisibilityString()); + queryMap.set(PrivateAuditConstants.COLUMN_VISIBILITY, marking.toAccessExpressionString()); queryMap.set(PrivateAuditConstants.AUDIT_TYPE, auditType.name()); queryMap.set(PrivateAuditConstants.USER_DN, q.getUserDN()); queryMap.set(PrivateAuditConstants.LOGIC_CLASS, logic.getLogicName()); @@ -1310,7 +1310,7 @@ public CachedResultsResponse create(@Required("queryId") @PathParam("queryId") S MultiValueMap params = new LinkedMultiValueMap<>(query.toMap()); marking.validate(params); PrivateAuditConstants.stripPrivateParameters(queryParameters); - params.set(PrivateAuditConstants.COLUMN_VISIBILITY, marking.toColumnVisibilityString()); + params.set(PrivateAuditConstants.COLUMN_VISIBILITY, marking.toAccessExpressionString()); params.set(PrivateAuditConstants.AUDIT_TYPE, auditType.name()); params.set(PrivateAuditConstants.USER_DN, query.getUserDN()); params.set(PrivateAuditConstants.LOGIC_CLASS, crq.getQueryLogic().getLogicName()); diff --git a/web-services/client/src/main/java/datawave/webservice/query/cachedresults/CacheableQueryRow.java b/web-services/client/src/main/java/datawave/webservice/query/cachedresults/CacheableQueryRow.java index b3157341f62..d01668bc0f9 100644 --- a/web-services/client/src/main/java/datawave/webservice/query/cachedresults/CacheableQueryRow.java +++ b/web-services/client/src/main/java/datawave/webservice/query/cachedresults/CacheableQueryRow.java @@ -8,6 +8,7 @@ import java.util.Set; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.webservice.query.result.event.HasMarkings; import datawave.webservice.query.util.TypedValue; @@ -27,15 +28,15 @@ public abstract class CacheableQueryRow implements HasMarkings { fixedColumnSet.add("_column_timestamps_"); } - protected MarkingFunctions markingFunctions; + protected MarkingFunctions markingFunctions; public static Set getFixedColumnSet() { return Collections.unmodifiableSet(fixedColumnSet); } - public abstract void addColumn(String columnName, String columnStringValue, Map markings, String columnVisibility, Long timestamp); + public abstract void addColumn(String columnName, String columnStringValue, Markings markings, String columnVisibility, Long timestamp); - public abstract void addColumn(String columnName, TypedValue columnValue, Map markings, String columnVisibility, Long timestamp); + public abstract void addColumn(String columnName, TypedValue columnValue, Markings markings, String columnVisibility, Long timestamp); public abstract String getUser(); @@ -51,7 +52,7 @@ public static Set getFixedColumnSet() { public abstract String getColFam(); - public abstract Map getColumnMarkings(String columnName); + public abstract Markings getColumnMarkings(String columnName); public abstract String getColumnVisibility(String columnName); @@ -98,17 +99,17 @@ protected static String createColumnList(Set columnNames, Map> columnMarkingsMap); + public abstract void setColumnMarkingsMap(Map> columnMarkingsMap); public abstract void setColumnColumnVisibilityMap(Map columnVisibilityMap); public abstract void setColumnTimestampMap(Map parseColumnTimestamps); - public MarkingFunctions getMarkingFunctions() { + public MarkingFunctions getMarkingFunctions() { return markingFunctions; } - public void setMarkingFunctions(MarkingFunctions markingFunctions) { + public void setMarkingFunctions(MarkingFunctions markingFunctions) { this.markingFunctions = markingFunctions; } } diff --git a/web-services/client/src/main/java/datawave/webservice/query/result/EdgeQueryResponseBase.java b/web-services/client/src/main/java/datawave/webservice/query/result/EdgeQueryResponseBase.java index ac2c03acbe9..8419d0664e8 100644 --- a/web-services/client/src/main/java/datawave/webservice/query/result/EdgeQueryResponseBase.java +++ b/web-services/client/src/main/java/datawave/webservice/query/result/EdgeQueryResponseBase.java @@ -1,11 +1,11 @@ package datawave.webservice.query.result; import java.util.List; -import java.util.Map; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; +import datawave.marking.Markings; import datawave.webservice.query.result.edge.EdgeBase; import datawave.webservice.query.result.event.HasMarkings; import datawave.webservice.result.BaseQueryResponse; @@ -13,7 +13,7 @@ @XmlAccessorType(XmlAccessType.NONE) public abstract class EdgeQueryResponseBase extends BaseQueryResponse implements HasMarkings { - protected Map markings; + protected Markings markings; private static final long serialVersionUID = 5957852042933196910L; diff --git a/web-services/client/src/main/java/datawave/webservice/query/result/edge/DefaultEdge.java b/web-services/client/src/main/java/datawave/webservice/query/result/edge/DefaultEdge.java index d535e7009e9..357ea392559 100644 --- a/web-services/client/src/main/java/datawave/webservice/query/result/edge/DefaultEdge.java +++ b/web-services/client/src/main/java/datawave/webservice/query/result/edge/DefaultEdge.java @@ -5,7 +5,6 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; -import java.util.Map; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -17,6 +16,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; +import datawave.marking.Markings; import datawave.webservice.query.util.OptionallyEncodedStringAdapter; import io.protostuff.Input; import io.protostuff.Message; @@ -30,7 +30,7 @@ public class DefaultEdge implements EdgeBase, Serializable, Message private static final long serialVersionUID = -1621626861385080614L; - protected Map markings; + protected Markings markings; // The visibility of this edge @XmlElement(name = "ColumnVisibility", required = true, nillable = false) @@ -208,12 +208,12 @@ public void setActivityDate(String activityDate) { } @Override - public void setMarkings(Map markings) { + public void setMarkings(Markings markings) { this.markings = markings; - this.setColumnVisibility(markings.get(COLUMN_VISIBILITY)); } - public Map getMarkings() { + @Override + public Markings getMarkings() { return this.markings; } diff --git a/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultEvent.java b/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultEvent.java index 5b535e044be..c546ca47231 100644 --- a/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultEvent.java +++ b/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultEvent.java @@ -5,7 +5,6 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; -import java.util.Map; import javax.xml.bind.annotation.XmlAccessOrder; import javax.xml.bind.annotation.XmlAccessType; @@ -14,12 +13,10 @@ import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlElementWrapper; import javax.xml.bind.annotation.XmlTransient; -import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; import org.apache.commons.lang.builder.EqualsBuilder; import datawave.webservice.query.data.ObjectSizeOf; -import datawave.webservice.xml.util.StringMapAdapter; import io.protostuff.Input; import io.protostuff.Message; import io.protostuff.Output; @@ -31,10 +28,6 @@ public class DefaultEvent extends EventBase implement private static final long serialVersionUID = 2390592143914560317L; - @XmlElement(name = "Markings") - @XmlJavaTypeAdapter(StringMapAdapter.class) - private HashMap markings = null; - @XmlElement(name = "Metadata") private Metadata metadata = null; @@ -55,24 +48,6 @@ public String toString() { return getMarkings() + ": " + (this.fields != null ? this.fields.toString() : "fields are null"); } - @Override - public Map getMarkings() { - if (markings != null) { - return markings; - } else { - return super.getMarkings(); - } - } - - public void setMarkings(Map markings) { - if (null != markings) { - this.markings = new HashMap<>(markings); - } else { - this.markings = null; - } - super.setMarkings(this.markings); - } - public Metadata getMetadata() { return metadata; } @@ -88,7 +63,7 @@ public boolean equals(Object o) { EqualsBuilder eb = new EqualsBuilder(); - eb.append(this.markings, v.markings); + eb.append(this.getMarkings(), v.getMarkings()); eb.append(this.metadata, v.metadata); eb.append(this.fields, v.fields); return eb.isEquals(); @@ -99,7 +74,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - int result = markings != null ? markings.hashCode() : 0; + int result = getMarkings() != null ? getMarkings().hashCode() : 0; result = 31 * result + (metadata != null ? metadata.hashCode() : 0); result = 31 * result + (fields != null ? fields.hashCode() : 0); return result; @@ -133,9 +108,6 @@ public boolean isInitialized(DefaultEvent message) { } public void writeTo(Output output, DefaultEvent message) throws IOException { - if (message.markings != null) - output.writeObject(1, message.markings, MapSchema.SCHEMA, false); - if (message.metadata != null) { output.writeObject(2, message.metadata, Metadata.getSchema(), false); } @@ -159,8 +131,9 @@ public void mergeFrom(Input input, DefaultEvent message) throws IOException { while ((number = input.readFieldNumber(this)) != 0) { switch (number) { case 1: - message.markings = new HashMap(); - input.mergeObject(message.markings, MapSchema.SCHEMA); + // legacy markings map — consume but discard + HashMap legacyMarkings = new HashMap<>(); + input.mergeObject(legacyMarkings, MapSchema.SCHEMA); break; case 2: message.metadata = input.mergeObject(null, Metadata.getSchema()); diff --git a/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultFacets.java b/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultFacets.java index b0a1416b85b..b11be50cf80 100644 --- a/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultFacets.java +++ b/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultFacets.java @@ -4,7 +4,6 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; -import java.util.Map; import javax.xml.bind.annotation.XmlAccessOrder; import javax.xml.bind.annotation.XmlAccessType; @@ -12,6 +11,7 @@ import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlTransient; +import datawave.marking.Markings; import datawave.webservice.query.data.ObjectSizeOf; import io.protostuff.Input; import io.protostuff.Message; @@ -24,11 +24,13 @@ public class DefaultFacets extends FacetsBase implements Serializable, Message markings) { + @Override + public void setMarkings(Markings markings) { this.markings = markings; } - public Map getMarkings() { + @Override + public Markings getMarkings() { return this.markings; } diff --git a/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultField.java b/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultField.java index 86c78a21b7e..1d4445b9139 100644 --- a/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultField.java +++ b/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultField.java @@ -3,7 +3,6 @@ import java.io.IOException; import java.io.Serializable; import java.util.HashMap; -import java.util.Map; import javax.xml.bind.annotation.XmlAccessOrder; import javax.xml.bind.annotation.XmlAccessType; @@ -12,7 +11,6 @@ import javax.xml.bind.annotation.XmlAttribute; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlTransient; -import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; @@ -20,8 +18,8 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import datawave.data.type.Type; +import datawave.marking.Markings; import datawave.webservice.query.util.TypedValue; -import datawave.webservice.xml.util.StringMapAdapter; import io.protostuff.Input; import io.protostuff.Message; import io.protostuff.Output; @@ -38,9 +36,6 @@ public class DefaultField extends FieldBase implements Serializabl private static final long serialVersionUID = -3982566563059126017L; - @XmlElement(name = "Markings") - @XmlJavaTypeAdapter(StringMapAdapter.class) - private HashMap markings; @XmlAttribute(name = "columnVisibility") private String columnVisibility; @XmlAttribute(name = "timestamp") @@ -52,10 +47,10 @@ public class DefaultField extends FieldBase implements Serializabl public DefaultField() {} - public DefaultField(String name, String columnVisibility, Map markings, Long timestamp, Object value) { + public DefaultField(String name, String columnVisibility, Markings markings, Long timestamp, Object value) { super(); this.name = name; - this.markings = new HashMap(markings); + this.markings = markings; this.columnVisibility = columnVisibility; this.timestamp = timestamp; this.value = new TypedValue(value); @@ -69,17 +64,12 @@ public DefaultField(String name, String columnVisibility, Long timestamp, Object this.value = new TypedValue(value); } - /* - * (non-Javadoc) - * - * @see datawave.webservice.query.result.event.FieldInterface#setMarkings(java.util.Map) - */ @Override - public void setMarkings(Map markings) { - this.markings = new HashMap(markings); + public void setMarkings(Markings markings) { + this.markings = markings; } - public Map getMarkings() { + public Markings getMarkings() { return markings; } @@ -145,8 +135,7 @@ public String getValueString() { @Override public String toString() { StringBuilder buf = new StringBuilder(); - buf.append("Field [markings=").append(markings); - buf.append(" columnVisibility=").append(columnVisibility); + buf.append("Field [columnVisibility=").append(columnVisibility); buf.append(" timestamp=").append(timestamp); buf.append(" name=").append(name); buf.append(" value= "); @@ -159,7 +148,7 @@ public String toString() { @Override public int hashCode() { - return new HashCodeBuilder(17, 37).append(markings).append(columnVisibility).append(timestamp).append(name).append(value).hashCode(); + return new HashCodeBuilder(17, 37).append(columnVisibility).append(timestamp).append(name).append(value).hashCode(); } @Override @@ -171,7 +160,6 @@ public boolean equals(Object o) { eb.append(this.name, v.name); eb.append(this.timestamp, v.timestamp); - eb.append(this.markings, v.markings); eb.append(this.columnVisibility, v.columnVisibility); eb.append(this.value, v.value); return eb.isEquals(); @@ -215,8 +203,6 @@ public boolean isInitialized(DefaultField message) { @Override public void writeTo(Output output, DefaultField message) throws IOException { - if (message.markings != null) - output.writeObject(1, message.markings, MapSchema.SCHEMA, false); if (message.columnVisibility != null) output.writeString(2, message.columnVisibility, false); output.writeUInt64(3, message.timestamp, false); @@ -232,8 +218,9 @@ public void mergeFrom(Input input, DefaultField message) throws IOException { while ((number = input.readFieldNumber(this)) != 0) { switch (number) { case 1: - message.markings = new HashMap(); - input.mergeObject(message.markings, MapSchema.SCHEMA); + // legacy markings map — skip but consume + HashMap legacyMarkings = new HashMap<>(); + input.mergeObject(legacyMarkings, MapSchema.SCHEMA); break; case 2: message.columnVisibility = input.readString(); diff --git a/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultFieldCardinality.java b/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultFieldCardinality.java index acb724af967..df5557c61fd 100644 --- a/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultFieldCardinality.java +++ b/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultFieldCardinality.java @@ -3,7 +3,6 @@ import java.io.IOException; import java.io.Serializable; import java.util.HashMap; -import java.util.Map; import javax.xml.bind.annotation.XmlAccessOrder; import javax.xml.bind.annotation.XmlAccessType; @@ -16,6 +15,7 @@ import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; +import datawave.marking.Markings; import io.protostuff.Input; import io.protostuff.Message; import io.protostuff.Output; @@ -65,11 +65,12 @@ public void setField(String field) { } @Override - public void setMarkings(Map markings) { + public void setMarkings(Markings markings) { this.markings = markings; } - public Map getMarkings() { + @Override + public Markings getMarkings() { return this.markings; } diff --git a/web-services/client/src/main/java/datawave/webservice/query/result/event/EventBase.java b/web-services/client/src/main/java/datawave/webservice/query/result/event/EventBase.java index 5314c8f1b0a..3eb5668ef22 100644 --- a/web-services/client/src/main/java/datawave/webservice/query/result/event/EventBase.java +++ b/web-services/client/src/main/java/datawave/webservice/query/result/event/EventBase.java @@ -1,22 +1,20 @@ package datawave.webservice.query.result.event; import java.util.List; -import java.util.Map; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlSeeAlso; import javax.xml.bind.annotation.XmlTransient; -import com.google.common.collect.Maps; - +import datawave.marking.Markings; import io.protostuff.Message; @XmlAccessorType(XmlAccessType.NONE) @XmlSeeAlso(DefaultEvent.class) public abstract class EventBase> implements HasMarkings, Message { - protected transient Map markings; + protected transient Markings markings; protected transient boolean intermediateResult; @@ -50,17 +48,11 @@ public abstract class EventBase> implements HasMarkings @XmlTransient public abstract List getFields(); - public Map getMarkings() { - assureMarkings(); + public Markings getMarkings() { return markings; } - protected void assureMarkings() { - if (this.markings == null) - this.markings = Maps.newHashMap(); - } - - public void setMarkings(Map markings) { + public void setMarkings(Markings markings) { this.markings = markings; } diff --git a/web-services/client/src/main/java/datawave/webservice/query/result/event/FacetsBase.java b/web-services/client/src/main/java/datawave/webservice/query/result/event/FacetsBase.java index d5f6423dc4d..b01735a7f38 100644 --- a/web-services/client/src/main/java/datawave/webservice/query/result/event/FacetsBase.java +++ b/web-services/client/src/main/java/datawave/webservice/query/result/event/FacetsBase.java @@ -1,7 +1,6 @@ package datawave.webservice.query.result.event; import java.util.List; -import java.util.Map; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -10,7 +9,9 @@ import javax.xml.bind.annotation.XmlSeeAlso; import javax.xml.bind.annotation.XmlTransient; -import org.apache.accumulo.core.security.ColumnVisibility; +import org.apache.accumulo.access.AccessExpression; + +import datawave.marking.Markings; /** * @@ -19,14 +20,14 @@ @XmlSeeAlso(DefaultFacets.class) public abstract class FacetsBase implements HasMarkings { - protected Map markings; + protected Markings markings; @XmlElementWrapper(name = "Fields") @XmlElement(name = "Field") protected List fields = null; @XmlTransient - protected ColumnVisibility columnVisibility; + protected AccessExpression accessExpression; @SuppressWarnings("unchecked") public void setFields(List fields) { @@ -37,12 +38,12 @@ public List getFields() { return fields; } - public ColumnVisibility getColumnVisibility() { - return columnVisibility; + public AccessExpression getAccessExpression() { + return accessExpression; } - public void setColumnVisibility(ColumnVisibility columnVisibility) { - this.columnVisibility = columnVisibility; + public void setAccessExpression(AccessExpression accessExpression) { + this.accessExpression = accessExpression; } public abstract void setSizeInBytes(long sizeInBytes); diff --git a/web-services/client/src/main/java/datawave/webservice/query/result/event/FieldBase.java b/web-services/client/src/main/java/datawave/webservice/query/result/event/FieldBase.java index ad2fb2d0dfe..a3119cd2530 100644 --- a/web-services/client/src/main/java/datawave/webservice/query/result/event/FieldBase.java +++ b/web-services/client/src/main/java/datawave/webservice/query/result/event/FieldBase.java @@ -1,15 +1,12 @@ package datawave.webservice.query.result.event; -import java.util.Map; - import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlSeeAlso; -import org.apache.accumulo.core.security.ColumnVisibility; - -import com.google.common.base.Charsets; +import org.apache.accumulo.access.AccessExpression; +import datawave.marking.Markings; import datawave.webservice.query.util.TypedValue; import io.protostuff.Message; @@ -17,7 +14,7 @@ @XmlSeeAlso(DefaultField.class) public abstract class FieldBase implements HasMarkings, Message { - protected transient Map markings; + protected transient Markings markings; public abstract Long getTimestamp(); @@ -39,9 +36,9 @@ public abstract class FieldBase implements HasMarkings, Message { public abstract void setColumnVisibility(String columnVisibility); - public void setColumnVisibility(ColumnVisibility columnVisibility) { - String cvString = (columnVisibility == null) ? null : new String(columnVisibility.getExpression(), Charsets.UTF_8); - setColumnVisibility(cvString); + public void setColumnVisibility(AccessExpression accessExpression) { + String exprString = (accessExpression == null) ? null : accessExpression.getExpression(); + setColumnVisibility(exprString); } } diff --git a/web-services/client/src/main/java/datawave/webservice/query/result/event/FieldCardinalityBase.java b/web-services/client/src/main/java/datawave/webservice/query/result/event/FieldCardinalityBase.java index 62ca15fed8c..73bfc3125d9 100644 --- a/web-services/client/src/main/java/datawave/webservice/query/result/event/FieldCardinalityBase.java +++ b/web-services/client/src/main/java/datawave/webservice/query/result/event/FieldCardinalityBase.java @@ -1,14 +1,12 @@ package datawave.webservice.query.result.event; -import java.util.Map; - import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlSeeAlso; -import org.apache.accumulo.core.security.ColumnVisibility; +import org.apache.accumulo.access.AccessExpression; -import com.google.common.base.Charsets; +import datawave.marking.Markings; /** * @@ -17,15 +15,15 @@ @XmlSeeAlso(DefaultFieldCardinality.class) public abstract class FieldCardinalityBase implements HasMarkings { - protected Map markings; + protected Markings markings; public abstract String getColumnVisibility(); public abstract void setColumnVisibility(String columnVisibility); - public void setColumnVisibility(ColumnVisibility columnVisibility) { - String cvString = (columnVisibility == null) ? null : new String(columnVisibility.getExpression(), Charsets.UTF_8); - setColumnVisibility(cvString); + public void setColumnVisibility(AccessExpression accessExpression) { + String exprString = (accessExpression == null) ? null : accessExpression.getExpression(); + setColumnVisibility(exprString); } public abstract String getField(); diff --git a/web-services/client/src/main/java/datawave/webservice/result/DefaultEdgeQueryResponse.java b/web-services/client/src/main/java/datawave/webservice/result/DefaultEdgeQueryResponse.java index e54d29659a9..04db73661b2 100644 --- a/web-services/client/src/main/java/datawave/webservice/result/DefaultEdgeQueryResponse.java +++ b/web-services/client/src/main/java/datawave/webservice/result/DefaultEdgeQueryResponse.java @@ -6,7 +6,6 @@ import java.util.Collections; import java.util.LinkedList; import java.util.List; -import java.util.Map; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -15,6 +14,7 @@ import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlTransient; +import datawave.marking.Markings; import datawave.webservice.query.exception.QueryExceptionType; import datawave.webservice.query.result.EdgeQueryResponseBase; import datawave.webservice.query.result.edge.DefaultEdge; @@ -53,12 +53,13 @@ public String getSecurityMarkings() { return securityMarkings; } - public void setMarkings(Map markings) { + @Override + public void setMarkings(Markings markings) { this.markings = markings; - this.setSecurityMarkings(markings.get("security")); } - public Map getMarkings() { + @Override + public Markings getMarkings() { return markings; } diff --git a/web-services/client/src/main/java/datawave/webservice/result/FacetQueryResponse.java b/web-services/client/src/main/java/datawave/webservice/result/FacetQueryResponse.java index 82da40f2863..c69149202f7 100644 --- a/web-services/client/src/main/java/datawave/webservice/result/FacetQueryResponse.java +++ b/web-services/client/src/main/java/datawave/webservice/result/FacetQueryResponse.java @@ -6,7 +6,6 @@ import java.util.Collections; import java.util.LinkedList; import java.util.List; -import java.util.Map; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -16,6 +15,7 @@ import javax.xml.bind.annotation.XmlTransient; import javax.xml.bind.annotation.XmlType; +import datawave.marking.Markings; import datawave.webservice.query.exception.QueryExceptionType; import datawave.webservice.query.result.event.DefaultFacets; import datawave.webservice.query.result.event.FacetsBase; @@ -46,11 +46,13 @@ public FacetQueryResponse(List facets) { this.facets = new ArrayList(facets); } - public void setMarkings(Map markings) { + @Override + public void setMarkings(Markings markings) { this.markings = markings; } - public Map getMarkings() { + @Override + public Markings getMarkings() { return markings; } diff --git a/web-services/client/src/main/java/datawave/webservice/result/FacetQueryResponseBase.java b/web-services/client/src/main/java/datawave/webservice/result/FacetQueryResponseBase.java index 27426102282..bbe2a288b40 100644 --- a/web-services/client/src/main/java/datawave/webservice/result/FacetQueryResponseBase.java +++ b/web-services/client/src/main/java/datawave/webservice/result/FacetQueryResponseBase.java @@ -1,11 +1,11 @@ package datawave.webservice.result; import java.util.List; -import java.util.Map; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; +import datawave.marking.Markings; import datawave.webservice.query.result.event.FacetsBase; import datawave.webservice.query.result.event.HasMarkings; @@ -16,7 +16,7 @@ public abstract class FacetQueryResponseBase extends BaseQueryResponse implements HasMarkings { private static final long serialVersionUID = -3483112784845232037L; - protected transient Map markings; + protected transient Markings markings; public abstract Long getTotalEvents(); @@ -27,6 +27,4 @@ public abstract class FacetQueryResponseBase extends BaseQueryResponse implement public abstract void setFacets(List facets); public abstract void addFacet(FacetsBase facetInterface); - - public abstract void setMarkings(Map markings); } diff --git a/web-services/client/src/main/java/datawave/webservice/result/keyword/DefaultTagCloud.java b/web-services/client/src/main/java/datawave/webservice/result/keyword/DefaultTagCloud.java index 4b20f2f12de..48457cf59ec 100644 --- a/web-services/client/src/main/java/datawave/webservice/result/keyword/DefaultTagCloud.java +++ b/web-services/client/src/main/java/datawave/webservice/result/keyword/DefaultTagCloud.java @@ -31,10 +31,6 @@ public class DefaultTagCloud extends TagCloudBase markings = null; - @XmlElement(name = "language") private String language = null; @@ -46,24 +42,6 @@ public class DefaultTagCloud extends TagCloudBase tags = null; - @Override - public Map getMarkings() { - if (markings != null) { - return markings; - } else { - return super.getMarkings(); - } - } - - public void setMarkings(Map markings) { - if (null != markings) { - this.markings = new HashMap<>(markings); - } else { - this.markings = null; - } - super.setMarkings(this.markings); - } - @Override public String getLanguage() { return language; @@ -120,9 +98,6 @@ public boolean isInitialized(DefaultTagCloud message) { } public void writeTo(Output output, DefaultTagCloud message) throws IOException { - if (message.markings != null) - output.writeObject(1, message.markings, MapSchema.SCHEMA, false); - if (message.language != null) { output.writeString(2, message.language, false); } @@ -150,8 +125,9 @@ public void mergeFrom(Input input, DefaultTagCloud message) throws IOException { while ((number = input.readFieldNumber(this)) != 0) { switch (number) { case 1: - message.markings = new HashMap(); - input.mergeObject(message.markings, MapSchema.SCHEMA); + // legacy markings map — consume but discard + HashMap legacyMarkings = new HashMap<>(); + input.mergeObject(legacyMarkings, MapSchema.SCHEMA); break; case 2: message.language = input.readString(); @@ -217,13 +193,13 @@ public boolean equals(Object other) { } DefaultTagCloud otherCloud = (DefaultTagCloud) other; - return Objects.equals(this.markings, otherCloud.markings) && Objects.equals(this.language, otherCloud.language) + return Objects.equals(this.getMarkings(), otherCloud.getMarkings()) && Objects.equals(this.language, otherCloud.language) && Objects.equals(this.metadata, otherCloud.metadata) && this.tags.size() == otherCloud.tags.size() && new HashSet<>(this.tags).containsAll(otherCloud.tags); } @Override public int hashCode() { - return Objects.hash(markings, language, metadata, tags); + return Objects.hash(getMarkings(), language, metadata, tags); } } diff --git a/web-services/client/src/main/java/datawave/webservice/result/keyword/TagCloudBase.java b/web-services/client/src/main/java/datawave/webservice/result/keyword/TagCloudBase.java index 8840a7ad185..412f0e3a849 100644 --- a/web-services/client/src/main/java/datawave/webservice/result/keyword/TagCloudBase.java +++ b/web-services/client/src/main/java/datawave/webservice/result/keyword/TagCloudBase.java @@ -7,28 +7,23 @@ import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlSeeAlso; -import com.google.common.collect.Maps; - +import datawave.marking.Markings; import datawave.webservice.query.result.event.HasMarkings; import io.protostuff.Message; @XmlAccessorType(XmlAccessType.NONE) @XmlSeeAlso(DefaultTagCloud.class) public abstract class TagCloudBase> implements HasMarkings, Message { - protected transient Map markings; + protected transient Markings markings; protected transient boolean intermediateResult; - public Map getMarkings() { - assureMarkings(); + @Override + public Markings getMarkings() { return markings; } - protected void assureMarkings() { - if (this.markings == null) - this.markings = Maps.newHashMap(); - } - - public void setMarkings(Map markings) { + @Override + public void setMarkings(Markings markings) { this.markings = markings; } diff --git a/web-services/metrics/src/main/java/datawave/query/metrics/ShardTableQueryMetricHandler.java b/web-services/metrics/src/main/java/datawave/query/metrics/ShardTableQueryMetricHandler.java index 33a48fd211a..7510bee7c2f 100644 --- a/web-services/metrics/src/main/java/datawave/query/metrics/ShardTableQueryMetricHandler.java +++ b/web-services/metrics/src/main/java/datawave/query/metrics/ShardTableQueryMetricHandler.java @@ -75,7 +75,6 @@ import datawave.ingest.mapreduce.job.BulkIngestKey; import datawave.ingest.mapreduce.job.writer.LiveContextWriter; import datawave.ingest.table.config.TableConfigHelper; -import datawave.marking.MarkingFunctions; import datawave.microservice.query.Query; import datawave.microservice.query.QueryImpl; import datawave.microservice.query.QueryImpl.Parameter; @@ -283,8 +282,9 @@ private Multimap getEntries(AbstractColumnBasedHandler event.setVisibility(new ColumnVisibility(DEFAULT_SECURITY_MARKING)); } else { try { - MarkingFunctions markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); - event.setVisibility(markingFunctions.translateToColumnVisibility(markings)); + // Extract the visibility expression from the legacy Map markings + String visExpr = markings.getOrDefault("columnVisibility", DEFAULT_SECURITY_MARKING); + event.setVisibility(new ColumnVisibility(visExpr)); } catch (Exception e) { log.error(e.getMessage(), e); event.setVisibility(new ColumnVisibility(DEFAULT_SECURITY_MARKING)); diff --git a/web-services/query/src/main/java/datawave/webservice/query/factory/Persister.java b/web-services/query/src/main/java/datawave/webservice/query/factory/Persister.java index 6549a0a2c93..61892c38162 100644 --- a/web-services/query/src/main/java/datawave/webservice/query/factory/Persister.java +++ b/web-services/query/src/main/java/datawave/webservice/query/factory/Persister.java @@ -107,7 +107,7 @@ public Query create(String userDN, List dnList, SecurityMarking marking, MultivaluedMap optionalQueryParameters) { Query q = responseObjectFactory.getQueryImpl(); q.initialize(userDN, dnList, queryLogicName, qp, MapUtils.toMultiValueMap(optionalQueryParameters)); - q.setColumnVisibility(marking.toColumnVisibilityString()); + q.setColumnVisibility(marking.toAccessExpressionString()); q.setUncaughtExceptionHandler(new QueryUncaughtExceptionHandler()); Thread.currentThread().setUncaughtExceptionHandler(q.getUncaughtExceptionHandler()); // Persist the query object if required diff --git a/web-services/query/src/main/java/datawave/webservice/query/runner/QueryExecutorBean.java b/web-services/query/src/main/java/datawave/webservice/query/runner/QueryExecutorBean.java index 718f5af012f..c98940db3c3 100644 --- a/web-services/query/src/main/java/datawave/webservice/query/runner/QueryExecutorBean.java +++ b/web-services/query/src/main/java/datawave/webservice/query/runner/QueryExecutorBean.java @@ -616,7 +616,7 @@ private QueryData validateQueryParameters(String queryLogicName, UriInfo uriInfo // These are parameters that aren't passed in by the user, but rather are computed from other sources. PrivateAuditConstants.stripPrivateParameters(queryParameters); queryParameters.add(PrivateAuditConstants.LOGIC_CLASS, queryLogicName); - queryParameters.putSingle(PrivateAuditConstants.COLUMN_VISIBILITY, marking.toColumnVisibilityString()); + queryParameters.putSingle(PrivateAuditConstants.COLUMN_VISIBILITY, marking.toAccessExpressionString()); queryParameters.add(PrivateAuditConstants.USER_DN, qd.userDn); return qd; diff --git a/web-services/query/src/test/java/datawave/webservice/query/logic/composite/CompositeQueryLogicTest.java b/web-services/query/src/test/java/datawave/webservice/query/logic/composite/CompositeQueryLogicTest.java index 15e8a2597df..372d0f17322 100644 --- a/web-services/query/src/test/java/datawave/webservice/query/logic/composite/CompositeQueryLogicTest.java +++ b/web-services/query/src/test/java/datawave/webservice/query/logic/composite/CompositeQueryLogicTest.java @@ -41,6 +41,7 @@ import datawave.core.query.logic.composite.CompositeQueryLogic; import datawave.core.query.logic.filtered.FilteredQueryLogic; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.microservice.query.Query; import datawave.microservice.query.QueryImpl; import datawave.security.authorization.AuthorizationException; @@ -152,13 +153,13 @@ public long getTotalResults() { } @Override - public void setMarkings(Map markings) { + public void setMarkings(Markings markings) { this.markings = markings; } @Override - public Map getMarkings() { - return Collections.unmodifiableMap(markings); + public Markings getMarkings() { + return this.markings; } } From 091811d190d6b519008bc78dcf33f1649a39d731 Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Tue, 17 Mar 2026 18:11:17 -0400 Subject: [PATCH 03/63] compilation fixes Conflicts: microservices/services/config/pom.xml warehouse/query-core/src/main/java/datawave/query/tables/shard/CountAggregatingIterator.java # Conflicts: # microservices/services/accumulo/service/pom.xml # microservices/services/audit/service/pom.xml # microservices/services/authorization/service/pom.xml # microservices/services/config/pom.xml # microservices/services/dictionary/service/pom.xml # microservices/services/file-provider/service/pom.xml # microservices/services/hazelcast/client/pom.xml # microservices/services/hazelcast/common/pom.xml # microservices/services/hazelcast/service/pom.xml # microservices/services/modification/service/pom.xml # microservices/services/query-executor/service/pom.xml # microservices/services/query-metric/api/pom.xml # microservices/services/query-metric/service/pom.xml # microservices/services/query/service/pom.xml # microservices/starters/audit/pom.xml # microservices/starters/cached-results/pom.xml # microservices/starters/datawave/pom.xml # microservices/starters/metadata/pom.xml # microservices/starters/query-metric/pom.xml # microservices/starters/query/pom.xml --- .../CacheableQueryRowReader.java | 2 +- .../common/util/AccessExpressionUtil.java | 20 +++++++++++++ .../modification/MutableMetadataHandler.java | 6 ++-- .../core/query/logic/BaseQueryLogic.java | 6 ++-- .../logic/BaseQueryLogicTransformer.java | 4 +-- .../query/logic/DelegatingQueryLogic.java | 4 +-- .../datawave/core/query/logic/QueryLogic.java | 4 +-- .../FrequencyMetadataAggregator.java | 2 +- .../accumulo/admin/AdminService.java | 4 +-- .../config/AccumuloConfiguration.java | 2 +- .../accumulo/lookup/LookupService.java | 4 +-- .../dictionary/data/DescriptionBase.java | 17 ++++------- .../data/DefaultDescriptionTest.java | 29 +++++++++++-------- .../DictionaryServiceConfiguration.java | 4 +-- .../dictionary/data/DataDictionaryImpl.java | 4 +-- .../metadata/MetadataDescriptionsHelper.java | 4 +-- .../DataDictionaryControllerTest.java | 8 ++--- .../data/DataDictionaryImplTest.java | 2 +- .../querymetric/QueryMetricOperations.java | 9 +++--- .../QueryMetricHandlerConfiguration.java | 2 +- .../factory/QueryMetricQueryLogicFactory.java | 4 +-- .../LocalShardTableQueryMetricHandler.java | 2 +- .../RemoteShardTableQueryMetricHandler.java | 2 +- .../handler/ShardTableQueryMetricHandler.java | 11 ++++--- .../querymetric/QueryMetricTestBase.java | 10 +++++-- .../AlternateQueryMetricConfiguration.java | 2 +- ...AlternateShardTableQueryMetricHandler.java | 2 +- .../query/AbstractQueryServiceTest.java | 3 +- .../query/lookup/LookupServiceTest.java | 2 +- .../CachedResultsQueryService.java | 5 ++-- pom.xml | 11 +++++++ .../marking/MarkingFunctionsFactory.java | 6 ++-- .../data/config/ingest/CompositeIngest.java | 12 ++++++-- .../data/config/ingest/VirtualIngest.java | 12 ++++++-- .../ingest/mapreduce/EventMapper.java | 2 +- .../handler/atom/AtomDataTypeHandler.java | 2 +- .../atom/AtomErrorDataTypeHandler.java | 2 +- .../edge/ProtobufEdgeDataTypeHandler.java | 2 +- .../handler/error/ErrorDataTypeHandler.java | 2 +- .../mapreduce/handler/facet/FacetHandler.java | 2 +- .../handler/shard/ShardedDataTypeHandler.java | 2 +- .../handler/ssdeep/SSDeepIndexHandler.java | 2 +- .../datawave/query/attributes/Document.java | 4 +-- .../query/common/grouping/GroupingUtils.java | 14 ++++----- .../query/discovery/DiscoveryIterator.java | 2 +- .../query/discovery/DiscoveryTransformer.java | 2 +- .../query/iterator/GroupingIterator.java | 4 +-- .../query/predicate/ValueToAttributes.java | 6 ++-- .../table/parser/ContentKeyValueFactory.java | 2 +- .../table/parser/EventKeyValueFactory.java | 4 +-- .../table/parser/KeywordKeyValueFactory.java | 2 +- .../parser/TermFrequencyKeyValueFactory.java | 4 +-- .../query/tables/BaseRemoteQueryLogic.java | 2 +- .../query/tables/RemoteEdgeQueryLogic.java | 4 +-- .../query/tables/RemoteEventQueryLogic.java | 6 ++-- .../query/tables/ShardQueryLogic.java | 4 +-- .../shard/CountAggregatingIterator.java | 10 +++---- .../shard/CountResultPostprocessor.java | 4 +-- .../ssdeep/SSDeepDiscoveryQueryLogic.java | 2 +- .../SSDeepSimilarityQueryTransformer.java | 2 +- .../transformer/ContentQueryTransformer.java | 4 +-- .../query/transformer/DocumentTransform.java | 6 ++-- .../transformer/DocumentTransformer.java | 6 ++-- .../DocumentTransformerSupport.java | 10 +++---- .../transformer/EdgeQueryTransformer.java | 2 +- .../EdgeQueryTransformerSupport.java | 2 +- .../transformer/EventQueryTransformer.java | 4 +-- .../EventQueryTransformerSupport.java | 4 +-- .../query/transformer/FacetedTransformer.java | 10 +++---- .../FieldIndexCountQueryTransformer.java | 2 +- .../GroupingDocumentTransformer.java | 6 ++-- .../query/transformer/GroupingTransform.java | 2 +- .../ParentDocumentTransformer.java | 4 +-- .../ShardIndexQueryTransformer.java | 2 +- .../ShardQueryCountTableTransformer.java | 2 +- .../transformer/TagCloudTransformer.java | 2 +- .../TermFrequencyQueryTransformer.java | 2 +- .../annotation/AnnotationHitsTransformer.java | 2 +- .../query/util/HasMarkingFunctions.java | 4 +-- .../predicate/ValueToAttributesTest.java | 2 +- .../tables/ssdeep/SSDeepIngestQueryTest.java | 2 +- .../AnnotationHitsTransformerTest.java | 2 +- .../metrics/ShardTableQueryMetricHandler.java | 8 ++--- .../TestLegacyBaseQueryLogicTransformer.java | 2 +- .../composite/CompositeQueryLogicTest.java | 4 +-- 85 files changed, 229 insertions(+), 182 deletions(-) diff --git a/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowReader.java b/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowReader.java index a8a2eabd465..c65fd0bb004 100644 --- a/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowReader.java +++ b/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowReader.java @@ -24,7 +24,7 @@ public class CacheableQueryRowReader { @SuppressWarnings("unchecked") public static CacheableQueryRow createRow(ResultSet cachedRowSet, Set fixedFieldsInEvent, ResponseObjectFactory responseObjectFactory, - MarkingFunctions markingFunctions) { + MarkingFunctions markingFunctions) { CacheableQueryRow cqfc = responseObjectFactory.getCacheableQueryRow(); cqfc.setMarkingFunctions(markingFunctions); diff --git a/core/common-util/src/main/java/datawave/core/common/util/AccessExpressionUtil.java b/core/common-util/src/main/java/datawave/core/common/util/AccessExpressionUtil.java index 9216ccc9c30..379eceabb39 100644 --- a/core/common-util/src/main/java/datawave/core/common/util/AccessExpressionUtil.java +++ b/core/common-util/src/main/java/datawave/core/common/util/AccessExpressionUtil.java @@ -2,6 +2,8 @@ import static java.nio.charset.StandardCharsets.UTF_8; +import datawave.marking.AccessExpressionMarkings; +import datawave.marking.Markings; import org.apache.accumulo.access.Access; import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.security.ColumnVisibility; @@ -35,6 +37,24 @@ public static AccessExpression toAccessExpression(ColumnVisibility cv) { return ACCESS.newExpression(new String(expr, UTF_8)); } + /** + * Convert an {@link Markings} to a {@link ColumnVisibility}. + * + * @param markings + * the access expression + * @return the equivalent column visibility + */ + public static ColumnVisibility toColumnVisibility(Markings markings) { + if (markings instanceof AccessExpressionMarkings) { + return toColumnVisibility((AccessExpression) markings.getMarkings()); + } + String m = markings.toString(); + if (m.isEmpty()) { + return new ColumnVisibility(); + } + return new ColumnVisibility(m); + } + /** * Convert an {@link AccessExpression} to a {@link ColumnVisibility}. * diff --git a/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java b/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java index 397dee15488..9326aaceddc 100644 --- a/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java +++ b/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java @@ -206,7 +206,7 @@ public class MutableMetadataHandler extends ModificationServiceConfiguration { protected String reverseIndexTableName = null; protected String metadataTableName = null; protected MetadataHelperFactory metadataHelperFactory; - protected MarkingFunctions markingFunctions = null; + protected MarkingFunctions markingFunctions = null; // a map of event fields to index only/derived fields to enable appropriate deleting of event fields and all derivatives protected Multimap indexOnlyMap = null; @@ -249,11 +249,11 @@ public void setMetadataTableName(String metadataTableName) { this.metadataTableName = metadataTableName; } - public MarkingFunctions getMarkingFunctions() { + public MarkingFunctions getMarkingFunctions() { return markingFunctions; } - public void setMarkingFunctions(MarkingFunctions markingFunctions) { + public void setMarkingFunctions(MarkingFunctions markingFunctions) { this.markingFunctions = markingFunctions; } diff --git a/core/query/src/main/java/datawave/core/query/logic/BaseQueryLogic.java b/core/query/src/main/java/datawave/core/query/logic/BaseQueryLogic.java index d52daead974..d7b90715228 100644 --- a/core/query/src/main/java/datawave/core/query/logic/BaseQueryLogic.java +++ b/core/query/src/main/java/datawave/core/query/logic/BaseQueryLogic.java @@ -47,7 +47,7 @@ public abstract class BaseQueryLogic implements QueryLogic { protected ProxiedUserDetails serverUser; protected Set requiredRoles; - protected MarkingFunctions markingFunctions; + protected MarkingFunctions markingFunctions; protected ResponseObjectFactory responseObjectFactory; protected SelectorExtractor selectorExtractor; protected ResponseEnricherBuilder responseEnricherBuilder = null; @@ -100,11 +100,11 @@ public String getPlan(AccumuloClient client, Query settings, Set return settings.getQuery(); } - public MarkingFunctions getMarkingFunctions() { + public MarkingFunctions getMarkingFunctions() { return markingFunctions; } - public void setMarkingFunctions(MarkingFunctions markingFunctions) { + public void setMarkingFunctions(MarkingFunctions markingFunctions) { this.markingFunctions = markingFunctions; } diff --git a/core/query/src/main/java/datawave/core/query/logic/BaseQueryLogicTransformer.java b/core/query/src/main/java/datawave/core/query/logic/BaseQueryLogicTransformer.java index d7928802e41..43e2935394a 100644 --- a/core/query/src/main/java/datawave/core/query/logic/BaseQueryLogicTransformer.java +++ b/core/query/src/main/java/datawave/core/query/logic/BaseQueryLogicTransformer.java @@ -4,9 +4,9 @@ public abstract class BaseQueryLogicTransformer extends AbstractQueryLogicTransformer implements QueryLogicTransformer { - protected MarkingFunctions markingFunctions; + protected MarkingFunctions markingFunctions; - public BaseQueryLogicTransformer(MarkingFunctions markingFunctions) { + public BaseQueryLogicTransformer(MarkingFunctions markingFunctions) { if (null == markingFunctions) { throw new IllegalArgumentException("MarkingFunctions must be set"); } diff --git a/core/query/src/main/java/datawave/core/query/logic/DelegatingQueryLogic.java b/core/query/src/main/java/datawave/core/query/logic/DelegatingQueryLogic.java index 40a61167d9c..d28f60842fc 100644 --- a/core/query/src/main/java/datawave/core/query/logic/DelegatingQueryLogic.java +++ b/core/query/src/main/java/datawave/core/query/logic/DelegatingQueryLogic.java @@ -251,12 +251,12 @@ public String getConnPoolName() { } @Override - public MarkingFunctions getMarkingFunctions() { + public MarkingFunctions getMarkingFunctions() { return delegate.getMarkingFunctions(); } @Override - public void setMarkingFunctions(MarkingFunctions markingFunctions) { + public void setMarkingFunctions(MarkingFunctions markingFunctions) { delegate.setMarkingFunctions(markingFunctions); } diff --git a/core/query/src/main/java/datawave/core/query/logic/QueryLogic.java b/core/query/src/main/java/datawave/core/query/logic/QueryLogic.java index 0254faf2a54..929ffc04d6a 100644 --- a/core/query/src/main/java/datawave/core/query/logic/QueryLogic.java +++ b/core/query/src/main/java/datawave/core/query/logic/QueryLogic.java @@ -317,9 +317,9 @@ default String getResponseClass(Query query) throws QueryException { Set getRequiredRoles(); - MarkingFunctions getMarkingFunctions(); + MarkingFunctions getMarkingFunctions(); - void setMarkingFunctions(MarkingFunctions markingFunctions); + void setMarkingFunctions(MarkingFunctions markingFunctions); ResponseObjectFactory getResponseObjectFactory(); diff --git a/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java b/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java index 468fe974d80..94b74e7ef22 100644 --- a/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java +++ b/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java @@ -59,7 +59,7 @@ public class FrequencyMetadataAggregator extends WrappingIterator implements Opt private static final Logger log = Logger.getLogger(FrequencyMetadataAggregator.class); private static final String NULL_BYTE = "\0"; - private static final MarkingFunctions markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); + private static final MarkingFunctions markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); private boolean combineVisibilities; private String columnsOption; diff --git a/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/admin/AdminService.java b/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/admin/AdminService.java index 6ae90b00860..717f6965bd2 100644 --- a/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/admin/AdminService.java +++ b/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/admin/AdminService.java @@ -69,11 +69,11 @@ public class AdminService { private final Logger log = LoggerFactory.getLogger(this.getClass()); - private final MarkingFunctions markingFunctions; + private final MarkingFunctions markingFunctions; private final AccumuloClient warehouseAccumuloClient; @Autowired - public AdminService(@Qualifier("warehouse") AccumuloClient warehouseAccumuloClient, MarkingFunctions markingFunctions) { + public AdminService(@Qualifier("warehouse") AccumuloClient warehouseAccumuloClient, MarkingFunctions markingFunctions) { this.warehouseAccumuloClient = warehouseAccumuloClient; this.markingFunctions = markingFunctions; } diff --git a/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/config/AccumuloConfiguration.java b/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/config/AccumuloConfiguration.java index 7a4ac5cf8a4..48a0f79dcb0 100644 --- a/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/config/AccumuloConfiguration.java +++ b/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/config/AccumuloConfiguration.java @@ -37,7 +37,7 @@ public AccumuloProperties metricsAccumuloProperies(MetricsClusterProperties metr @Bean @ConditionalOnMissingBean - public MarkingFunctions markingFunctions() { + public MarkingFunctions markingFunctions() { return new MarkingFunctions.Default(); } diff --git a/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/lookup/LookupService.java b/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/lookup/LookupService.java index 578fe540a20..ee538a75b72 100644 --- a/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/lookup/LookupService.java +++ b/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/lookup/LookupService.java @@ -82,7 +82,7 @@ public interface Parameter { } private final SecurityMarking auditSecurityMarking; - private final MarkingFunctions markingFunctions; + private final MarkingFunctions markingFunctions; private final AccumuloClient connection; private final LookupAuditProperties lookupAuditProperties; private final LookupProperties lookupProperties; @@ -97,7 +97,7 @@ public interface Parameter { public LookupService( @Qualifier("auditLookupSecurityMarking") SecurityMarking auditSecurityMarking, - MarkingFunctions markingFunctions, + MarkingFunctions markingFunctions, @Qualifier("warehouse") AccumuloClient connection, LookupAuditProperties lookupAuditProperties, diff --git a/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java b/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java index bf6b534f882..10c181cfd33 100644 --- a/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java +++ b/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java @@ -10,6 +10,8 @@ import datawave.webservice.query.result.event.HasMarkings; import datawave.webservice.xml.util.StringMapAdapter; import io.protostuff.Message; +import lombok.Getter; +import lombok.Setter; @XmlAccessorType(XmlAccessType.NONE) public abstract class DescriptionBase implements HasMarkings, Message { @@ -19,22 +21,15 @@ public abstract class DescriptionBase implements HasMarkings, Message { /** * JAXB-serialized markings map. The {@link HasMarkings} interface methods {@link #getMarkings()}/{@link #setMarkings} operate on {@code Markings}; - * subclasses must bridge between this map and the Markings type. + * subclasses must bridge between this map and the Markings type. -- GETTER -- Direct access to the JAXB markings map. -- SETTER -- Direct access to set the + * JAXB markings map. */ + @Setter + @Getter @XmlElement(name = "markings") @XmlJavaTypeAdapter(StringMapAdapter.class) protected Map markingsMap; - /** Direct access to the JAXB markings map. */ - public Map getMarkingsMap() { - return markingsMap; - } - - /** Direct access to set the JAXB markings map. */ - public void setMarkingsMap(Map markingsMap) { - this.markingsMap = markingsMap; - } - public abstract String getDescription(); public abstract void setDescription(String description); diff --git a/microservices/services/dictionary/api/src/test/java/datawave/webservice/dictionary/data/DefaultDescriptionTest.java b/microservices/services/dictionary/api/src/test/java/datawave/webservice/dictionary/data/DefaultDescriptionTest.java index aed32592d10..fbb98bc2e4a 100644 --- a/microservices/services/dictionary/api/src/test/java/datawave/webservice/dictionary/data/DefaultDescriptionTest.java +++ b/microservices/services/dictionary/api/src/test/java/datawave/webservice/dictionary/data/DefaultDescriptionTest.java @@ -27,6 +27,22 @@ public class DefaultDescriptionTest { public void testMarshall() throws JAXBException { JAXBContext j = JAXBContext.newInstance(DefaultFields.class); + DefaultFields dicFieldsList = getDefaultFields(); + + Marshaller m = j.createMarshaller(); + m.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + m.marshal(dicFieldsList, out); + + String result = out.toString(); + + assertTrue(result.contains("columnVisibility")); + assertTrue(result.contains("PRIVATE")); + assertTrue(result.contains("my description")); + } + + private static DefaultFields getDefaultFields() { Map markings = new HashMap<>(); markings.put("columnVisibility", "PRIVATE"); @@ -47,18 +63,7 @@ public void testMarshall() throws JAXBException { DefaultFields dicFieldsList = new DefaultFields(); dicFieldsList.setFields(dicFields); - - Marshaller m = j.createMarshaller(); - m.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true); - - ByteArrayOutputStream out = new ByteArrayOutputStream(); - m.marshal(dicFieldsList, out); - - String result = out.toString(); - - assertTrue(result.contains("columnVisibility")); - assertTrue(result.contains("PRIVATE")); - assertTrue(result.contains("my description")); + return dicFieldsList; } @Test diff --git a/microservices/services/dictionary/service/src/main/java/datawave/microservice/dictionary/config/DictionaryServiceConfiguration.java b/microservices/services/dictionary/service/src/main/java/datawave/microservice/dictionary/config/DictionaryServiceConfiguration.java index 5cd8815170b..41bc312dc62 100644 --- a/microservices/services/dictionary/service/src/main/java/datawave/microservice/dictionary/config/DictionaryServiceConfiguration.java +++ b/microservices/services/dictionary/service/src/main/java/datawave/microservice/dictionary/config/DictionaryServiceConfiguration.java @@ -46,14 +46,14 @@ public UserAuthFunctions userAuthFunctions() { @Bean @Scope("prototype") @ConditionalOnMissingBean - public MetadataDescriptionsHelper metadataHelperWithDescriptions(MarkingFunctions markingFunctions, + public MetadataDescriptionsHelper metadataHelperWithDescriptions(MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { return new MetadataDescriptionsHelper<>(markingFunctions, responseObjectFactory); } @Bean @ConditionalOnMissingBean - public DataDictionary datawaveDataDictionary(MarkingFunctions markingFunctions, + public DataDictionary datawaveDataDictionary(MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, MetadataHelperFactory metadataHelperFactory, MetadataDescriptionsHelperFactory metadataDescriptionsHelperFactory) { return new DataDictionaryImpl(markingFunctions, responseObjectFactory, metadataHelperFactory, metadataDescriptionsHelperFactory); diff --git a/microservices/services/dictionary/service/src/main/java/datawave/microservice/dictionary/data/DataDictionaryImpl.java b/microservices/services/dictionary/service/src/main/java/datawave/microservice/dictionary/data/DataDictionaryImpl.java index 5a508b37406..d0042199493 100644 --- a/microservices/services/dictionary/service/src/main/java/datawave/microservice/dictionary/data/DataDictionaryImpl.java +++ b/microservices/services/dictionary/service/src/main/java/datawave/microservice/dictionary/data/DataDictionaryImpl.java @@ -32,14 +32,14 @@ public class DataDictionaryImpl implements DataDictionary { - private final MarkingFunctions markingFunctions; + private final MarkingFunctions markingFunctions; private final ResponseObjectFactory responseObjectFactory; private final MetadataHelperFactory metadataHelperFactory; private final MetadataDescriptionsHelperFactory metadataDescriptionsHelperFactory; private Map normalizationMap = Maps.newHashMap(); private String dataDictionarySystem = ""; - public DataDictionaryImpl(MarkingFunctions markingFunctions, + public DataDictionaryImpl(MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, MetadataHelperFactory metadataHelperFactory, MetadataDescriptionsHelperFactory metadataDescriptionsHelperFactory) { this.markingFunctions = markingFunctions; diff --git a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java index 1478210b176..0fc059d783d 100644 --- a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java +++ b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java @@ -50,14 +50,14 @@ public class MetadataDescriptionsHelper> { private static final Logger log = LoggerFactory.getLogger(MetadataDescriptionsHelper.class); - private final MarkingFunctions markingFunctions; + private final MarkingFunctions markingFunctions; private final ResponseObjectFactory responseObjectFactory; private String metadataTableName; private AccumuloClient accumuloClient; private Set fullUserAuths; - public MetadataDescriptionsHelper(MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { + public MetadataDescriptionsHelper(MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { this.markingFunctions = markingFunctions; this.responseObjectFactory = responseObjectFactory; } diff --git a/microservices/services/dictionary/service/src/test/java/datawave/microservice/dictionary/DataDictionaryControllerTest.java b/microservices/services/dictionary/service/src/test/java/datawave/microservice/dictionary/DataDictionaryControllerTest.java index 34f531566a8..1e5512f7840 100644 --- a/microservices/services/dictionary/service/src/test/java/datawave/microservice/dictionary/DataDictionaryControllerTest.java +++ b/microservices/services/dictionary/service/src/test/java/datawave/microservice/dictionary/DataDictionaryControllerTest.java @@ -31,9 +31,6 @@ import datawave.accumulo.inmemory.InMemoryAccumuloClient; import datawave.accumulo.inmemory.InMemoryInstance; -import datawave.marking.AccessExpressionMarkings; -import datawave.marking.MarkingFunctions; -import datawave.marking.Markings; import datawave.microservice.ControllerIT; import datawave.microservice.dictionary.config.DataDictionaryProperties; import datawave.webservice.dictionary.data.DefaultDataDictionary; @@ -130,10 +127,9 @@ public void testPostDescriptions() { .build(); // @formatter:on - Markings markings = AccessExpressionMarkings.builder().accessExpression(MarkingFunctions.Default.ACCESS.newExpression("USER|ADMIN")).build(); Multimap,DefaultDescription> descriptions = HashMultimap.create(); - descriptions.put(new AbstractMap.SimpleEntry<>("fooField", "fooType"), new DefaultDescription("my foo field", markings)); - descriptions.put(new AbstractMap.SimpleEntry<>("barField", "barType"), new DefaultDescription("my bar field", markings)); + descriptions.put(new AbstractMap.SimpleEntry<>("fooField", "fooType"), new DefaultDescription("my foo field", Map.of("columnVisibility", "USER"))); + descriptions.put(new AbstractMap.SimpleEntry<>("barField", "barType"), new DefaultDescription("my bar field", Map.of("columnVisibility", "ADMIN"))); DefaultFields postBody = new DefaultFields(descriptions); MultiValueMap additionalHeaders = new LinkedMultiValueMap<>(); additionalHeaders.set(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE); diff --git a/microservices/services/dictionary/service/src/test/java/datawave/microservice/dictionary/data/DataDictionaryImplTest.java b/microservices/services/dictionary/service/src/test/java/datawave/microservice/dictionary/data/DataDictionaryImplTest.java index 1fadd63ab54..6613613cb0d 100644 --- a/microservices/services/dictionary/service/src/test/java/datawave/microservice/dictionary/data/DataDictionaryImplTest.java +++ b/microservices/services/dictionary/service/src/test/java/datawave/microservice/dictionary/data/DataDictionaryImplTest.java @@ -59,7 +59,7 @@ public class DataDictionaryImplTest { private AccumuloClient accumuloClient; @Mock - private MarkingFunctions markingFunctions; + private MarkingFunctions markingFunctions; @Mock private ResponseObjectFactory responseObjectFactory; diff --git a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java index f2733f82d1b..2aa2f0526c3 100644 --- a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java +++ b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java @@ -25,6 +25,7 @@ import javax.annotation.PreDestroy; import javax.annotation.security.PermitAll; +import datawave.core.common.util.AccessExpressionUtil; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.accumulo.core.security.VisibilityEvaluator; @@ -87,7 +88,7 @@ public class QueryMetricOperations { private QueryGeometryHandler geometryHandler; private CacheManager cacheManager; private Cache incomingQueryMetricsCache; - private MarkingFunctions markingFunctions; + private MarkingFunctions markingFunctions; private QueryMetricResponseFactory queryMetricResponseFactory; private MergeLockLifecycleListener mergeLock; private Correlator correlator; @@ -141,7 +142,7 @@ enum DEFAULT_DATETIME { */ @Autowired public QueryMetricOperations(@Qualifier("queryMetricCacheManager") CacheManager cacheManager, ShardTableQueryMetricHandler handler, - QueryGeometryHandler geometryHandler, MarkingFunctions markingFunctions, QueryMetricResponseFactory queryMetricResponseFactory, + QueryGeometryHandler geometryHandler, MarkingFunctions markingFunctions, QueryMetricResponseFactory queryMetricResponseFactory, MergeLockLifecycleListener mergeLock, Correlator correlator, MetricUpdateEntryProcessorFactory entryProcessorFactory, QueryMetricOperationsStats stats, QueryMetricClient queryMetricClient, DnUtils dnUtils) { this.handler = handler; @@ -478,11 +479,11 @@ private boolean isMetricsAdministrator(DatawaveUserDetails currentUser) { private boolean canViewMetric(DatawaveUserDetails currentUser, BaseQueryMetric metric) throws Exception { boolean userCanViewMetric = true; - ColumnVisibility columnVisibility = this.markingFunctions.toAccessExpression(metric.getMarkings()); + ColumnVisibility columnVisibility = AccessExpressionUtil.toColumnVisibility(markingFunctions.toAccessExpression(metric.getMarkings())); for (DatawaveUser user : currentUser.getProxiedUsers()) { Authorizations authorizations = new Authorizations(user.getAuths().toArray(new String[0])); VisibilityEvaluator visibilityEvaluator = new VisibilityEvaluator(authorizations); - if (visibilityEvaluator.evaluate(columnVisibility) == false) { + if (!visibilityEvaluator.evaluate(columnVisibility)) { userCanViewMetric = false; break; } diff --git a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricHandlerConfiguration.java b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricHandlerConfiguration.java index a11c1ea1619..c7606028280 100644 --- a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricHandlerConfiguration.java +++ b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricHandlerConfiguration.java @@ -99,7 +99,7 @@ QueryMetricFactory queryMetricFactory() { @ConditionalOnMissingBean public ShardTableQueryMetricHandler shardTableQueryMetricHandler(QueryMetricHandlerProperties queryMetricHandlerProperties, AccumuloConnectionFactory connectionFactory, QueryMetricQueryLogicFactory logicFactory, QueryMetricFactory metricFactory, - MarkingFunctions markingFunctions, QueryMetricCombiner queryMetricCombiner, LuceneToJexlQueryParser luceneToJexlQueryParser, + MarkingFunctions markingFunctions, QueryMetricCombiner queryMetricCombiner, LuceneToJexlQueryParser luceneToJexlQueryParser, ResponseObjectFactory responseObjectFactory, WebClient.Builder webClientBuilder, @Autowired(required = false) JWTTokenHandler jwtTokenHandler, DnUtils dnUtils, QueryMetricResponseFactory queryMetricResponseFactory) { ShardTableQueryMetricHandler handler; diff --git a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/factory/QueryMetricQueryLogicFactory.java b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/factory/QueryMetricQueryLogicFactory.java index 68f8038704b..63ce5a661df 100644 --- a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/factory/QueryMetricQueryLogicFactory.java +++ b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/factory/QueryMetricQueryLogicFactory.java @@ -16,14 +16,14 @@ @Component public class QueryMetricQueryLogicFactory implements FactoryBean { - private MarkingFunctions markingFunctions; + private MarkingFunctions markingFunctions; private ResponseObjectFactory responseObjectFactory; private QueryMetricHandlerProperties queryMetricHandlerProperties; private DateIndexHelperFactory dateIndexHelperFactory; private MetadataHelperFactory metadataHelperFactory; @Autowired - public QueryMetricQueryLogicFactory(MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, + public QueryMetricQueryLogicFactory(MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, DateIndexHelperFactory dateIndexHelperFactory, MetadataHelperFactory metadataHelperFactory, QueryMetricHandlerProperties queryMetricHandlerProperties) { this.markingFunctions = markingFunctions; diff --git a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/LocalShardTableQueryMetricHandler.java b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/LocalShardTableQueryMetricHandler.java index 90b776d72bd..1a159f89789 100644 --- a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/LocalShardTableQueryMetricHandler.java +++ b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/LocalShardTableQueryMetricHandler.java @@ -48,7 +48,7 @@ public class LocalShardTableQueryMetricHandler extend protected ExecutorService executorService; public LocalShardTableQueryMetricHandler(QueryMetricHandlerProperties queryMetricHandlerProperties, AccumuloConnectionFactory connectionFactory, - QueryMetricQueryLogicFactory logicFactory, QueryMetricFactory metricFactory, MarkingFunctions markingFunctions, + QueryMetricQueryLogicFactory logicFactory, QueryMetricFactory metricFactory, MarkingFunctions markingFunctions, QueryMetricCombiner queryMetricCombiner, LuceneToJexlQueryParser luceneToJexlQueryParser, DnUtils dnUtils) { super(queryMetricHandlerProperties, connectionFactory, logicFactory, metricFactory, markingFunctions, queryMetricCombiner, luceneToJexlQueryParser, dnUtils); diff --git a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/RemoteShardTableQueryMetricHandler.java b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/RemoteShardTableQueryMetricHandler.java index 81eae0285ce..8381fdde6df 100644 --- a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/RemoteShardTableQueryMetricHandler.java +++ b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/RemoteShardTableQueryMetricHandler.java @@ -39,7 +39,7 @@ public class RemoteShardTableQueryMetricHandler exten private final JWTTokenHandler jwtTokenHandler; public RemoteShardTableQueryMetricHandler(QueryMetricHandlerProperties queryMetricHandlerProperties, AccumuloConnectionFactory connectionFactory, - QueryMetricQueryLogicFactory logicFactory, QueryMetricFactory metricFactory, MarkingFunctions markingFunctions, + QueryMetricQueryLogicFactory logicFactory, QueryMetricFactory metricFactory, MarkingFunctions markingFunctions, QueryMetricCombiner queryMetricCombiner, LuceneToJexlQueryParser luceneToJexlQueryParser, ResponseObjectFactory responseObjectFactory, WebClient.Builder webClientBuilder, JWTTokenHandler jwtTokenHandler, DnUtils dnUtils) { super(queryMetricHandlerProperties, connectionFactory, logicFactory, metricFactory, markingFunctions, queryMetricCombiner, luceneToJexlQueryParser, diff --git a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java index d911af6ebe0..f877a9a9cb1 100644 --- a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java +++ b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java @@ -19,6 +19,9 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantReadWriteLock; +import datawave.core.common.util.AccessExpressionUtil; +import datawave.marking.Markings; +import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; @@ -106,13 +109,13 @@ public abstract class ShardTableQueryMetricHandler ex protected QueryMetricFactory metricFactory; protected UIDBuilder uidBuilder = UID.builder(); protected QueryMetricCombiner queryMetricCombiner; - protected MarkingFunctions markingFunctions; + protected MarkingFunctions markingFunctions; protected DnUtils dnUtils; // this lock is necessary for when there is an error condition and the accumuloRecordWriter needs to be replaced protected ReentrantReadWriteLock accumuloRecordWriterLock = new ReentrantReadWriteLock(); public ShardTableQueryMetricHandler(QueryMetricHandlerProperties queryMetricHandlerProperties, AccumuloConnectionFactory connectionFactory, - QueryMetricQueryLogicFactory logicFactory, QueryMetricFactory metricFactory, MarkingFunctions markingFunctions, + QueryMetricQueryLogicFactory logicFactory, QueryMetricFactory metricFactory, MarkingFunctions markingFunctions, QueryMetricCombiner queryMetricCombiner, LuceneToJexlQueryParser luceneToJexlQueryParser, DnUtils dnUtils) { super(luceneToJexlQueryParser); this.queryMetricHandlerProperties = queryMetricHandlerProperties; @@ -365,10 +368,10 @@ protected Multimap getEntries(ContentIndexingColumnBasedHan event.setDataType(type); event.setTimestamp(updatedQueryMetric.getCreateDate().getTime()); // get markings from metric, otherwise use the default markings - Map markings = updatedQueryMetric.getMarkings(); + Markings markings = updatedQueryMetric.getMarkings(); if (markings != null && !markings.isEmpty()) { try { - event.setVisibility(this.markingFunctions.toAccessExpression(updatedQueryMetric.getMarkings())); + event.setVisibility(AccessExpressionUtil.toColumnVisibility(markingFunctions.toAccessExpression(markings))); } catch (MarkingFunctions.Exception e) { log.error(e.getMessage(), e); event.setVisibility(this.queryMetricHandlerProperties.getDefaultMetricVisibility()); diff --git a/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java b/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java index 04f830e87f8..d65064d0c9b 100644 --- a/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java +++ b/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java @@ -19,6 +19,10 @@ import java.util.Map; import java.util.Set; +import datawave.marking.AccessExpressionMarkings; +import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; +import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.BatchScanner; import org.apache.accumulo.core.client.BatchWriter; @@ -144,14 +148,14 @@ public class QueryMetricTestBase { protected DatawaveUserDetails nonAdminUser; protected static boolean isHazelCast; protected static CacheManager staticCacheManager; - protected static Map metricMarkings; + protected static Markings metricMarkings; protected List tables; protected Collection auths; protected AccumuloClient accumuloClient; static { - metricMarkings = new HashMap<>(); - metricMarkings.put(ColumnVisibilitySecurityMarking.VISIBILITY_MARKING, "A&C"); + AccessExpression ae = MarkingFunctions.Default.ACCESS.newExpression("A&C"); + metricMarkings = AccessExpressionMarkings.builder().accessExpression(ae).build(); } @AfterAll diff --git a/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/config/AlternateQueryMetricConfiguration.java b/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/config/AlternateQueryMetricConfiguration.java index 3695bf7602e..eb856d51e77 100644 --- a/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/config/AlternateQueryMetricConfiguration.java +++ b/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/config/AlternateQueryMetricConfiguration.java @@ -48,7 +48,7 @@ public BaseQueryMetric createMetric(boolean populateVersionMap) { @Bean public ShardTableQueryMetricHandler shardTableQueryMetricHandler(QueryMetricHandlerProperties queryMetricHandlerProperties, AccumuloConnectionFactory connectionFactory, QueryMetricQueryLogicFactory logicFactory, QueryMetricFactory metricFactory, - MarkingFunctions markingFunctions, QueryMetricCombiner queryMetricCombiner, LuceneToJexlQueryParser luceneToJexlQueryParser, + MarkingFunctions markingFunctions, QueryMetricCombiner queryMetricCombiner, LuceneToJexlQueryParser luceneToJexlQueryParser, DnUtils dnUtils) { return new AlternateShardTableQueryMetricHandler(queryMetricHandlerProperties, connectionFactory, logicFactory, metricFactory, markingFunctions, queryMetricCombiner, luceneToJexlQueryParser, dnUtils); diff --git a/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/config/AlternateShardTableQueryMetricHandler.java b/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/config/AlternateShardTableQueryMetricHandler.java index ff985dc0bd8..8f71e434bf2 100644 --- a/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/config/AlternateShardTableQueryMetricHandler.java +++ b/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/config/AlternateShardTableQueryMetricHandler.java @@ -18,7 +18,7 @@ public class AlternateShardTableQueryMetricHandler extends LocalShardTableQueryMetricHandler { public AlternateShardTableQueryMetricHandler(QueryMetricHandlerProperties queryMetricHandlerProperties, AccumuloConnectionFactory connectionFactory, - QueryMetricQueryLogicFactory logicFactory, QueryMetricFactory metricFactory, MarkingFunctions markingFunctions, + QueryMetricQueryLogicFactory logicFactory, QueryMetricFactory metricFactory, MarkingFunctions markingFunctions, QueryMetricCombiner queryMetricCombiner, LuceneToJexlQueryParser luceneToJexlQueryParser, DnUtils dnUtils) { super(queryMetricHandlerProperties, connectionFactory, logicFactory, metricFactory, markingFunctions, queryMetricCombiner, luceneToJexlQueryParser, dnUtils); diff --git a/microservices/services/query/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java b/microservices/services/query/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java index 80e9812d0bd..9aa72368f5d 100644 --- a/microservices/services/query/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java +++ b/microservices/services/query/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java @@ -27,6 +27,7 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import datawave.marking.MarkingFunctions; import org.apache.http.NameValuePair; import org.apache.http.client.utils.URLEncodedUtils; import org.junit.jupiter.api.AfterEach; @@ -168,7 +169,7 @@ protected void publishEventsToQueue(String queryId, int numEvents, MultiValueMap List fields = new ArrayList<>(); for (Map.Entry> entry : fieldValues.entrySet()) { for (String value : entry.getValue()) { - fields.add(new DefaultField(entry.getKey(), visibility, new HashMap<>(), currentTime, value)); + fields.add(new DefaultField(entry.getKey(), visibility, null, currentTime, value)); } } event.setFields(fields); diff --git a/microservices/services/query/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java b/microservices/services/query/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java index 35d7213d22c..124a3d14398 100644 --- a/microservices/services/query/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java +++ b/microservices/services/query/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java @@ -877,7 +877,7 @@ protected void publishEventsToQueue(String queryId, int numEvents, MultiValueMap List fields = new ArrayList<>(); for (Map.Entry> entry : fieldValues.entrySet()) { for (String value : entry.getValue()) { - fields.add(new DefaultField(entry.getKey(), visibility, new HashMap<>(), currentTime, value)); + fields.add(new DefaultField(entry.getKey(), visibility, null, currentTime, value)); } } event.setFields(fields); diff --git a/microservices/starters/cached-results/src/main/java/datawave/microservice/query/cachedresults/CachedResultsQueryService.java b/microservices/starters/cached-results/src/main/java/datawave/microservice/query/cachedresults/CachedResultsQueryService.java index 24b90230e09..3ce13342183 100644 --- a/microservices/starters/cached-results/src/main/java/datawave/microservice/query/cachedresults/CachedResultsQueryService.java +++ b/microservices/starters/cached-results/src/main/java/datawave/microservice/query/cachedresults/CachedResultsQueryService.java @@ -21,7 +21,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.TreeMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -112,7 +111,7 @@ public class CachedResultsQueryService { private final QueryLogicFactory queryLogicFactory; private final QueryStorageCache queryStorageCache; private final ResponseObjectFactory responseObjectFactory; - private final MarkingFunctions markingFunctions; + private final MarkingFunctions markingFunctions; // Note: CachedResultsQueryParameters needs to be request scoped private final RequestScopeBeanSupplier scopedCachedResultsQueryParameters; private final String fieldDefinitions; @@ -123,7 +122,7 @@ public class CachedResultsQueryService { public CachedResultsQueryService(CachedResultsQueryProperties cachedResultsQueryProperties, JdbcTemplate cachedResultsJdbcTemplate, CachedResultsQueryCache cachedResultsQueryCache, QueryService queryService, AuditClient auditClient, SecurityMarking securityMarking, QueryLogicFactory queryLogicFactory, QueryStorageCache queryStorageCache, ResponseObjectFactory responseObjectFactory, - MarkingFunctions markingFunctions, CachedResultsQueryParameters cachedResultsQueryParameters) { + MarkingFunctions markingFunctions, CachedResultsQueryParameters cachedResultsQueryParameters) { this.cachedResultsQueryProperties = cachedResultsQueryProperties; this.cachedResultsJdbcTemplate = cachedResultsJdbcTemplate; this.cachedResultsQueryCache = cachedResultsQueryCache; diff --git a/pom.xml b/pom.xml index 80a0f990dfd..ed1064f965d 100644 --- a/pom.xml +++ b/pom.xml @@ -79,10 +79,21 @@ 4.0.3-SNAPSHOT 4.0.3-SNAPSHOT 4.0.7-SNAPSHOT + 4.0.5-SNAPSHOT + 4.0.5-SNAPSHOT 1.0.2-SNAPSHOT 3.0.1-SNAPSHOT + 1.0.2-SNAPSHOT 1.0.2-SNAPSHOT + 1.0.10-SNAPSHOT 4.1.4-SNAPSHOT + 4.0.8-SNAPSHOT + 4.0.6-SNAPSHOT + 4.0.3-SNAPSHOT + 1.0.13-SNAPSHOT + 3.0.5-SNAPSHOT + 1.0.11-SNAPSHOT + 3.0.6-SNAPSHOT 1.9.0 3.6.3 5.2.0 diff --git a/warehouse/core/src/main/java/datawave/marking/MarkingFunctionsFactory.java b/warehouse/core/src/main/java/datawave/marking/MarkingFunctionsFactory.java index 256d44750a4..0f2fd62de79 100644 --- a/warehouse/core/src/main/java/datawave/marking/MarkingFunctionsFactory.java +++ b/warehouse/core/src/main/java/datawave/marking/MarkingFunctionsFactory.java @@ -21,13 +21,13 @@ public class MarkingFunctionsFactory { @Inject @SpringBean(refreshable = true) - private static MarkingFunctions applicationMarkingFunctions; + private static MarkingFunctions applicationMarkingFunctions; public static final Logger log = LoggerFactory.getLogger(MarkingFunctionsFactory.class); - private static MarkingFunctions markingFunctions; + private static MarkingFunctions markingFunctions; - public static synchronized MarkingFunctions createMarkingFunctions() { + public static synchronized MarkingFunctions createMarkingFunctions() { if (markingFunctions != null) return markingFunctions; diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CompositeIngest.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CompositeIngest.java index d607ed7a1dd..bfd12d9e412 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CompositeIngest.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CompositeIngest.java @@ -14,6 +14,7 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; +import datawave.marking.AccessExpressionMarkings; import org.apache.commons.lang.NotImplementedException; import org.apache.hadoop.conf.Configuration; import org.slf4j.Logger; @@ -132,7 +133,7 @@ class CompositeFieldNormalizer { protected Map allowMissing = new HashMap<>(); private Set ignoreNormalizationForFields = new HashSet<>(); - protected MarkingFunctions markingFunctions; + protected MarkingFunctions markingFunctions; public void setup(Type type, Configuration config) { @@ -295,14 +296,19 @@ private String[] getConfPrefixes(String type) { * different markings * @return the merged markings */ - @SuppressWarnings("unchecked") protected Markings mergeMarkings(Markings markings1, Markings markings2) { if (markings2 != null) { if (markings1 == null) { markings1 = markings2; } else { try { - markings1 = markingFunctions.combine(markings1, markings2); + if (markings1 instanceof AccessExpressionMarkings && markings2 instanceof AccessExpressionMarkings) { + AccessExpressionMarkings aem1 = (AccessExpressionMarkings) markings1; + AccessExpressionMarkings aem2 = (AccessExpressionMarkings) markings2; + return markingFunctions.fromAccessExpression(markingFunctions.combine(Set.of(aem1.getMarkings(), aem2.getMarkings()))); + } else { + throw new RuntimeException(String.format("Unknown markings class %s or %s", markings1.getClass().getName(), markings1.getClass().getName())); + } } catch (MarkingFunctions.Exception e) { throw new RuntimeException("Unable to combine markings.", e); } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java index 119eec308e1..6bec75acdef 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java @@ -13,6 +13,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import datawave.marking.AccessExpressionMarkings; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.builder.HashCodeBuilder; import org.apache.commons.lang3.builder.EqualsBuilder; @@ -127,7 +128,7 @@ class VirtualFieldNormalizer extends NoOpType { protected transient Map allowMissing = new HashMap<>(); private Set ignoreNormalizationForFields = new HashSet<>(); - private transient MarkingFunctions markingFunctions; + private transient MarkingFunctions markingFunctions; public void setup(Type type, String instance, Configuration config) { @@ -348,14 +349,19 @@ private String[] getConfPrefixes(String type, String instance) { * different markings * @return the merged markings */ - @SuppressWarnings("unchecked") protected Markings mergeMarkings(Markings markings1, Markings markings2) { if (markings2 != null) { if (markings1 == null) { markings1 = markings2; } else { try { - markings1 = markingFunctions.combine(markings1, markings2); + if (markings1 instanceof AccessExpressionMarkings && markings2 instanceof AccessExpressionMarkings) { + AccessExpressionMarkings aem1 = (AccessExpressionMarkings) markings1; + AccessExpressionMarkings aem2 = (AccessExpressionMarkings) markings2; + return markingFunctions.fromAccessExpression(markingFunctions.combine(Set.of(aem1.getMarkings(), aem2.getMarkings()))); + } else { + throw new RuntimeException(String.format("Unknown markings class %s or %s", markings1.getClass().getName(), markings1.getClass().getName())); + } } catch (MarkingFunctions.Exception e) { throw new RuntimeException("Unable to combine markings.", e); } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/EventMapper.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/EventMapper.java index d0ccd56b113..89be13c7b63 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/EventMapper.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/EventMapper.java @@ -161,7 +161,7 @@ public class EventMapper extends StatsDE protected Set sequenceFileNames = new HashSet<>(); - protected MarkingFunctions markingFunctions; + protected MarkingFunctions markingFunctions; private boolean metricsEnabled = false; private MetricsService metricsService; diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomDataTypeHandler.java index 86ea38b88bf..486ec19201d 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomDataTypeHandler.java @@ -65,7 +65,7 @@ public class AtomDataTypeHandler implements ExtendedDataT protected String[] fieldOverrides = null; protected HashMap> subCategories; protected String[] sCategories = null; - protected MarkingFunctions markingFunctions; + protected MarkingFunctions markingFunctions; protected Configuration conf; diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomErrorDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomErrorDataTypeHandler.java index d6b26e0cd6a..47c04df0ba1 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomErrorDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomErrorDataTypeHandler.java @@ -54,7 +54,7 @@ public class AtomErrorDataTypeHandler extends AtomDataTyp private AccessExpression defaultVisibility = null; private Configuration conf = null; private ErrorShardedIngestHelper errorHelper = null; - private MarkingFunctions markingFunctions; + private MarkingFunctions markingFunctions; @Override public void setup(TaskAttemptContext context) { diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDataTypeHandler.java index ca82592aebe..fb8fa5e1a73 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDataTypeHandler.java @@ -146,7 +146,7 @@ public class ProtobufEdgeDataTypeHandler implements Exten protected String edgeTableName = null; protected String metadataTableName = null; protected boolean enableMetadata = false; - protected MarkingFunctions markingFunctions; + protected MarkingFunctions markingFunctions; protected Map edges = null; diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorDataTypeHandler.java index 841e4e1f4df..52dfc0f6fcc 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorDataTypeHandler.java @@ -111,7 +111,7 @@ public class ErrorDataTypeHandler implements ExtendedData private byte[] defaultVisibility = null; protected MarkingsHelper markingsHelper; - protected MarkingFunctions markingFunctions; + protected MarkingFunctions markingFunctions; private String tableName = null; diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/facet/FacetHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/facet/FacetHandler.java index 75f14794c36..dc86577dd46 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/facet/FacetHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/facet/FacetHandler.java @@ -94,7 +94,7 @@ public class FacetHandler implements ExtendedDataTypeHand /* Instance variables */ - protected MarkingFunctions markingFunctions; + protected MarkingFunctions markingFunctions; protected ShardIdFactory shardIdFactory; protected TaskAttemptContext taskAttemptContext; diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java index c75ebd27173..d57146ac6de 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java @@ -223,7 +223,7 @@ public abstract class ShardedDataTypeHandler extends StatsDEnabledDataTyp private RawRecordMetadata metadata = null; private ShardIdFactory shardIdFactory = null; private LoadingCache dCache = null; - protected MarkingFunctions markingFunctions; + protected MarkingFunctions markingFunctions; protected IngestConfiguration ingestConfig = IngestConfigurationFactory.getIngestConfiguration(); // default setting is a standard index with uid and event keys diff --git a/warehouse/ingest-ssdeep/src/main/java/datawave/ingest/mapreduce/handler/ssdeep/SSDeepIndexHandler.java b/warehouse/ingest-ssdeep/src/main/java/datawave/ingest/mapreduce/handler/ssdeep/SSDeepIndexHandler.java index f91f8be1eb1..eff5f2c3ab0 100644 --- a/warehouse/ingest-ssdeep/src/main/java/datawave/ingest/mapreduce/handler/ssdeep/SSDeepIndexHandler.java +++ b/warehouse/ingest-ssdeep/src/main/java/datawave/ingest/mapreduce/handler/ssdeep/SSDeepIndexHandler.java @@ -69,7 +69,7 @@ public class SSDeepIndexHandler implements ExtendedDataTy protected Text ssdeepIndexTableName; - protected MarkingFunctions markingFunctions; + protected MarkingFunctions markingFunctions; protected TaskAttemptContext taskAttemptContext; diff --git a/warehouse/query-core/src/main/java/datawave/query/attributes/Document.java b/warehouse/query-core/src/main/java/datawave/query/attributes/Document.java index b398953957c..e2696ab998a 100644 --- a/warehouse/query-core/src/main/java/datawave/query/attributes/Document.java +++ b/warehouse/query-core/src/main/java/datawave/query/attributes/Document.java @@ -82,13 +82,13 @@ public Long load(Text row) { private static final long ONE_DAY_MS = 1000L * 60 * 60 * 24; - public MarkingFunctions getMarkingFunctions() { + public MarkingFunctions getMarkingFunctions() { return MarkingFunctions.Factory.createMarkingFunctions(); } public Markings getMarkings() { try { - MarkingFunctions markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); + MarkingFunctions markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); return markingFunctions.fromAccessExpression(AccessExpressionUtil.toAccessExpression(getColumnVisibility())); } catch (MarkingFunctions.Exception e) {} return null; diff --git a/warehouse/query-core/src/main/java/datawave/query/common/grouping/GroupingUtils.java b/warehouse/query-core/src/main/java/datawave/query/common/grouping/GroupingUtils.java index 5d20a47a6bf..647c8451144 100644 --- a/warehouse/query-core/src/main/java/datawave/query/common/grouping/GroupingUtils.java +++ b/warehouse/query-core/src/main/java/datawave/query/common/grouping/GroupingUtils.java @@ -44,7 +44,7 @@ public enum AverageAggregatorWriteFormat { * combined, it will be logged and a new, blank {@link ColumnVisibility} will be returned. * @return the combined column visibility */ - public static ColumnVisibility combineVisibilities(Collection expressions, MarkingFunctions markingFunctions, boolean failOnError) { + public static ColumnVisibility combineVisibilities(Collection expressions, MarkingFunctions markingFunctions, boolean failOnError) { try { return AccessExpressionUtil.toColumnVisibility(markingFunctions.combine(expressions)); } catch (MarkingFunctions.Exception e) { @@ -73,7 +73,7 @@ public static ColumnVisibility combineVisibilities(Collection * @return the new document */ @Deprecated - public static Document createDocument(Group group, List keys, MarkingFunctions markingFunctions, AverageAggregatorWriteFormat averageWriteFormat) { + public static Document createDocument(Group group, List keys, MarkingFunctions markingFunctions, AverageAggregatorWriteFormat averageWriteFormat) { Preconditions.checkState(!keys.isEmpty(), "No available keys for grouping results"); // Use the last (most recent) key so a new iterator will know where to start. @@ -94,7 +94,7 @@ public static Document createDocument(Group group, List keys, MarkingFuncti * the format to use when writing aggregated averages to the document * @return the new document */ - public static Document createDocument(Group group, Key key, MarkingFunctions markingFunctions, AverageAggregatorWriteFormat averageWriteFormat) { + public static Document createDocument(Group group, Key key, MarkingFunctions markingFunctions, AverageAggregatorWriteFormat averageWriteFormat) { Preconditions.checkNotNull(key, "document key cannot be null"); Document document = new Document(key, true); @@ -169,7 +169,7 @@ public static Document createDocument(Group group, Key key, MarkingFunctions mar * @param markingFunctions * the marking functions to use when combining column visibilities */ - private static void addSumAggregation(Document document, String field, SumAggregator aggregator, MarkingFunctions markingFunctions) { + private static void addSumAggregation(Document document, String field, SumAggregator aggregator, MarkingFunctions markingFunctions) { NumberType type = new NumberType(); type.setDelegate(aggregator.getAggregation()); TypeAttribute sumAttribute = new TypeAttribute<>(type, new Key(field + "_sum"), true); @@ -189,7 +189,7 @@ private static void addSumAggregation(Document document, String field, SumAggreg * @param markingFunctions * the marking functions to use when combining column visibilities */ - private static void addCountAggregation(Document document, String field, CountAggregator aggregator, MarkingFunctions markingFunctions) { + private static void addCountAggregation(Document document, String field, CountAggregator aggregator, MarkingFunctions markingFunctions) { NumberType type = new NumberType(); type.setDelegate(BigDecimal.valueOf(aggregator.getAggregation())); TypeAttribute sumAttribute = new TypeAttribute<>(type, new Key(field + "_count"), true); @@ -240,7 +240,7 @@ private static void addMaxAggregation(Document document, String field, MaxAggreg * @param markingFunctions * the marking functions to use when combining column visibilities */ - private static void addAverage(Document document, String field, AverageAggregator aggregator, MarkingFunctions markingFunctions) { + private static void addAverage(Document document, String field, AverageAggregator aggregator, MarkingFunctions markingFunctions) { NumberType type = new NumberType(); type.setDelegate(aggregator.getAggregation()); TypeAttribute attribute = new TypeAttribute<>(type, new Key(field + "_average"), true); @@ -260,7 +260,7 @@ private static void addAverage(Document document, String field, AverageAggregato * @param markingFunctions * the marking functions to use when combining column visibilities */ - private static void addAverageNumeratorAndDivisor(Document document, String field, AverageAggregator aggregator, MarkingFunctions markingFunctions) { + private static void addAverageNumeratorAndDivisor(Document document, String field, AverageAggregator aggregator, MarkingFunctions markingFunctions) { ColumnVisibility visibility = combineVisibilities(aggregator.getAccessExpressions(), markingFunctions, false); // Add an attribute for the average's numerator. This is required to properly combine additional aggregations in future groupings. diff --git a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java index ecfa21136ca..89e8054f884 100644 --- a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java +++ b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java @@ -38,7 +38,7 @@ public class DiscoveryIterator implements SortedKeyValueIterator { private static final Logger log = Logger.getLogger(DiscoveryIterator.class); - private static final MarkingFunctions markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); + private static final MarkingFunctions markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); private Key key; private Value value; diff --git a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryTransformer.java index 5bd0978854a..18c5d67b4bc 100644 --- a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryTransformer.java @@ -31,7 +31,7 @@ public class DiscoveryTransformer extends BaseQueryLogicTransformer variableFieldList = null; private BaseQueryLogic logic = null; private QueryModel myQueryModel = null; - private MarkingFunctions markingFunctions; + private MarkingFunctions markingFunctions; private ResponseObjectFactory responseObjectFactory; public DiscoveryTransformer(BaseQueryLogic logic, Query settings, QueryModel qm) { diff --git a/warehouse/query-core/src/main/java/datawave/query/iterator/GroupingIterator.java b/warehouse/query-core/src/main/java/datawave/query/iterator/GroupingIterator.java index 02a7921170d..ae0690e98c2 100644 --- a/warehouse/query-core/src/main/java/datawave/query/iterator/GroupingIterator.java +++ b/warehouse/query-core/src/main/java/datawave/query/iterator/GroupingIterator.java @@ -51,7 +51,7 @@ public class GroupingIterator implements Iterator> { // track the number of documents seen by this iterator private long documentCount = 0L; - private final MarkingFunctions markingFunctions; + private final MarkingFunctions markingFunctions; private final int groupFieldsBatchSize; @@ -61,7 +61,7 @@ public class GroupingIterator implements Iterator> { Map.Entry next; - public GroupingIterator(Iterator> previousIterators, MarkingFunctions markingFunctions, GroupFields groupFields, + public GroupingIterator(Iterator> previousIterators, MarkingFunctions markingFunctions, GroupFields groupFields, int groupFieldsBatchSize, YieldCallback yieldCallback) { this.previousIterators = previousIterators; this.markingFunctions = markingFunctions; diff --git a/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java b/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java index 668eba08655..79cb984e963 100644 --- a/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java +++ b/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java @@ -46,7 +46,7 @@ public class ValueToAttributes implements Function,Iterable> compositeToFieldMap; private Map> compositeFieldSeparatorsByType; - private final MarkingFunctions markingFunctions; + private final MarkingFunctions markingFunctions; private final Multimap> componentFieldToValues = ArrayListMultimap.create(); private final EventDataQueryFilter attrFilter; @@ -68,7 +68,7 @@ public class ValueToAttributes implements Function,Iterable markingFunctions, boolean fromIndex) { // in Java 25 we can create the attribute factory first this(new AttributeFactory(typeMetadata), compositeMetadata, attrFilter, markingFunctions, fromIndex); @@ -89,7 +89,7 @@ public ValueToAttributes(CompositeMetadata compositeMetadata, TypeMetadata typeM * flag denoting if this class is operating on values from the field index */ public ValueToAttributes(AttributeFactory attributeFactory, CompositeMetadata compositeMetadata, EventDataQueryFilter attrFilter, - MarkingFunctions markingFunctions, boolean fromIndex) { + MarkingFunctions markingFunctions, boolean fromIndex) { this.attrFactory = attributeFactory; this.markingFunctions = markingFunctions; this.attrFilter = attrFilter; diff --git a/warehouse/query-core/src/main/java/datawave/query/table/parser/ContentKeyValueFactory.java b/warehouse/query-core/src/main/java/datawave/query/table/parser/ContentKeyValueFactory.java index 8ed69bf246d..d2de6672545 100644 --- a/warehouse/query-core/src/main/java/datawave/query/table/parser/ContentKeyValueFactory.java +++ b/warehouse/query-core/src/main/java/datawave/query/table/parser/ContentKeyValueFactory.java @@ -19,7 +19,7 @@ public class ContentKeyValueFactory { private static final Logger log = Logger.getLogger(ContentKeyValueFactory.class); - public static ContentKeyValue parse(Key key, Value value, Authorizations auths, MarkingFunctions markingFunctions) throws MarkingFunctions.Exception { + public static ContentKeyValue parse(Key key, Value value, Authorizations auths, MarkingFunctions markingFunctions) throws MarkingFunctions.Exception { if (null == key) throw new IllegalArgumentException("Cannot pass null key to ContentKeyValueFactory"); diff --git a/warehouse/query-core/src/main/java/datawave/query/table/parser/EventKeyValueFactory.java b/warehouse/query-core/src/main/java/datawave/query/table/parser/EventKeyValueFactory.java index e977f9e4812..731d3698f6c 100644 --- a/warehouse/query-core/src/main/java/datawave/query/table/parser/EventKeyValueFactory.java +++ b/warehouse/query-core/src/main/java/datawave/query/table/parser/EventKeyValueFactory.java @@ -12,7 +12,7 @@ public class EventKeyValueFactory { - public static EventKeyValue parse(Key key, Value value, Authorizations auths, MarkingFunctions markingFunctions) throws MarkingFunctions.Exception { + public static EventKeyValue parse(Key key, Value value, Authorizations auths, MarkingFunctions markingFunctions) throws MarkingFunctions.Exception { if (null == key) throw new IllegalArgumentException("Cannot pass null key to EventKeyValueFactory"); @@ -39,7 +39,7 @@ public static EventKeyValue parse(Key key, Value value, Authorizations auths, Ma return e; } - protected static void parseColumnVisibility(EventKeyValue event, Key key, Authorizations auths, MarkingFunctions markingFunctions) + protected static void parseColumnVisibility(EventKeyValue event, Key key, Authorizations auths, MarkingFunctions markingFunctions) throws MarkingFunctions.Exception { event.setMarkings(markingFunctions.fromAccessExpression(AccessExpressionUtil.toAccessExpression(new ColumnVisibility(key.getColumnVisibility())))); } diff --git a/warehouse/query-core/src/main/java/datawave/query/table/parser/KeywordKeyValueFactory.java b/warehouse/query-core/src/main/java/datawave/query/table/parser/KeywordKeyValueFactory.java index 71a3fe315e2..0ee5ff66b45 100644 --- a/warehouse/query-core/src/main/java/datawave/query/table/parser/KeywordKeyValueFactory.java +++ b/warehouse/query-core/src/main/java/datawave/query/table/parser/KeywordKeyValueFactory.java @@ -12,7 +12,7 @@ /** Parses results returned from the KeywordExtractingIterator. Expects 'd' column keys, and serialized bytes as the value */ public class KeywordKeyValueFactory { - public static KeywordKeyValue parse(Key key, Value value, Authorizations auths, MarkingFunctions markingFunctions) throws MarkingFunctions.Exception { + public static KeywordKeyValue parse(Key key, Value value, Authorizations auths, MarkingFunctions markingFunctions) throws MarkingFunctions.Exception { if (null == key) throw new IllegalArgumentException("Cannot pass null key to KeywordKeyValueFactory"); diff --git a/warehouse/query-core/src/main/java/datawave/query/table/parser/TermFrequencyKeyValueFactory.java b/warehouse/query-core/src/main/java/datawave/query/table/parser/TermFrequencyKeyValueFactory.java index 80164dd823f..84a17312e9e 100644 --- a/warehouse/query-core/src/main/java/datawave/query/table/parser/TermFrequencyKeyValueFactory.java +++ b/warehouse/query-core/src/main/java/datawave/query/table/parser/TermFrequencyKeyValueFactory.java @@ -16,7 +16,7 @@ public class TermFrequencyKeyValueFactory { - public static TermFrequencyKeyValue parse(Key key, Value value, Authorizations auths, MarkingFunctions markingFunctions) throws MarkingFunctions.Exception { + public static TermFrequencyKeyValue parse(Key key, Value value, Authorizations auths, MarkingFunctions markingFunctions) throws MarkingFunctions.Exception { if (null == key) { throw new IllegalArgumentException("Cannot pass null key to TermFrequencyKeyValueFactory"); } @@ -55,7 +55,7 @@ public static TermFrequencyKeyValue parse(Key key, Value value, Authorizations a return t; } - protected static void parseColumnVisibility(TermFrequencyKeyValue tfkv, Key key, Authorizations auths, MarkingFunctions markingFunctions) + protected static void parseColumnVisibility(TermFrequencyKeyValue tfkv, Key key, Authorizations auths, MarkingFunctions markingFunctions) throws MarkingFunctions.Exception { tfkv.setMarkings(markingFunctions.fromAccessExpression(AccessExpressionUtil.toAccessExpression(key.getColumnVisibilityParsed()))); } diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/BaseRemoteQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/BaseRemoteQueryLogic.java index 2c7999e6665..8bf66fe9579 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/BaseRemoteQueryLogic.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/BaseRemoteQueryLogic.java @@ -130,7 +130,7 @@ public QueryLogicTransformer getTransformer(Query settings) { return transformerInstance; } - public abstract QueryLogicTransformer createTransformer(Query settings, MarkingFunctions markingFunctions, + public abstract QueryLogicTransformer createTransformer(Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory); @Override diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/RemoteEdgeQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/RemoteEdgeQueryLogic.java index 57df85a3b6a..7dfcad41b57 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/RemoteEdgeQueryLogic.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/RemoteEdgeQueryLogic.java @@ -58,7 +58,7 @@ public void setupQuery(GenericQueryConfiguration genericConfig) throws Exception } @Override - public QueryLogicTransformer createTransformer(Query settings, MarkingFunctions markingFunctions, + public QueryLogicTransformer createTransformer(Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { return new EdgeBaseTransformer(settings, markingFunctions, responseObjectFactory, edgeFields); } @@ -134,7 +134,7 @@ public EdgeModelFields getEdgeFields() { private class EdgeBaseTransformer extends EdgeQueryTransformerSupport { - public EdgeBaseTransformer(Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, EdgeModelFields fields) { + public EdgeBaseTransformer(Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, EdgeModelFields fields) { super(settings, markingFunctions, responseObjectFactory, fields); } diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/RemoteEventQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/RemoteEventQueryLogic.java index 3d466664fa7..f692c8dd104 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/RemoteEventQueryLogic.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/RemoteEventQueryLogic.java @@ -56,7 +56,7 @@ public void setupQuery(GenericQueryConfiguration genericConfig) throws Exception } @Override - public QueryLogicTransformer createTransformer(Query settings, MarkingFunctions markingFunctions, + public QueryLogicTransformer createTransformer(Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { return new EventBaseTransformer(settings, markingFunctions, responseObjectFactory); } @@ -124,11 +124,11 @@ public EventBase next() { private class EventBaseTransformer extends EventQueryTransformerSupport { - public EventBaseTransformer(Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { + public EventBaseTransformer(Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { super("notable", settings, markingFunctions, responseObjectFactory); } - public EventBaseTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, + public EventBaseTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { super(logic, settings, markingFunctions, responseObjectFactory); } diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java index 8be3c2e5482..f29be395a01 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java @@ -719,7 +719,7 @@ public QueryLogicTransformer getTransformer(Query settings) { return this.transformerInstance; } - MarkingFunctions markingFunctions = this.getMarkingFunctions(); + MarkingFunctions markingFunctions = this.getMarkingFunctions(); ResponseObjectFactory responseObjectFactory = this.getResponseObjectFactory(); boolean reducedInSettings = false; @@ -747,7 +747,7 @@ public QueryLogicTransformer getTransformer(Query settings) { return this.transformerInstance; } - protected DocumentTransformer createDocumentTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, + protected DocumentTransformer createDocumentTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, Boolean reducedResponse) { return new DocumentTransformer(logic, settings, markingFunctions, responseObjectFactory, reducedResponse); } diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountAggregatingIterator.java b/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountAggregatingIterator.java index 49668cf37d3..66cbf00a1dd 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountAggregatingIterator.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountAggregatingIterator.java @@ -27,7 +27,6 @@ import com.google.common.collect.Maps; import datawave.core.common.util.AccessExpressionUtil; -import datawave.core.iterators.ResultCountingIterator; import datawave.marking.MarkingFunctions; import datawave.webservice.query.result.event.DefaultEvent; @@ -60,7 +59,7 @@ public class CountAggregatingIterator extends TransformIterator { * @param markingFunctions * the marking functions */ - public CountAggregatingIterator(Iterator> iterator, Transformer transformer, MarkingFunctions markingFunctions) { + public CountAggregatingIterator(Iterator> iterator, Transformer transformer, MarkingFunctions markingFunctions) { this(iterator, transformer, markingFunctions, DEFAULT_PAGE_WAIT_TIME_MILLIS); } @@ -77,7 +76,8 @@ public CountAggregatingIterator(Iterator> iterator, Transformer * the time to wait for the next page */ @SuppressWarnings("unchecked") - public CountAggregatingIterator(Iterator> iterator, Transformer transformer, MarkingFunctions markingFunctions, long pageWaitTimeMillis) { + public CountAggregatingIterator(Iterator> iterator, Transformer transformer, MarkingFunctions markingFunctions, + long pageWaitTimeMillis) { super(iterator, transformer); this.aggregator = new CountEntryAggregator(transformer, markingFunctions); this.pageWaitTimeMillis = pageWaitTimeMillis; @@ -127,9 +127,9 @@ private static class CountEntryAggregator { private final Set columnVisibilities = new HashSet<>(); private final Transformer transformer; - private final MarkingFunctions markingFunctions; + private final MarkingFunctions markingFunctions; - public CountEntryAggregator(Transformer transformer, MarkingFunctions markingFunctions) { + public CountEntryAggregator(Transformer transformer, MarkingFunctions markingFunctions) { this.transformer = transformer; this.markingFunctions = markingFunctions; } diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountResultPostprocessor.java b/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountResultPostprocessor.java index ca3f16e4095..4ad6952936f 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountResultPostprocessor.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountResultPostprocessor.java @@ -20,9 +20,9 @@ public class CountResultPostprocessor implements ResultPostprocessor { private static final Logger log = Logger.getLogger(CountResultPostprocessor.class); - private final MarkingFunctions markingFunctions; + private final MarkingFunctions markingFunctions; - public CountResultPostprocessor(MarkingFunctions markingFunctions) { + public CountResultPostprocessor(MarkingFunctions markingFunctions) { this.markingFunctions = markingFunctions; } diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/ssdeep/SSDeepDiscoveryQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/ssdeep/SSDeepDiscoveryQueryLogic.java index 02a4d3c2dc7..f27ad4732ef 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/ssdeep/SSDeepDiscoveryQueryLogic.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/ssdeep/SSDeepDiscoveryQueryLogic.java @@ -209,7 +209,7 @@ public void setResponseObjectFactory(ResponseObjectFactory responseObjectFactory discoveryDelegate.setResponseObjectFactory(responseObjectFactory); } - public void setMarkingFunctions(MarkingFunctions markingFunctions) { + public void setMarkingFunctions(MarkingFunctions markingFunctions) { discoveryDelegate.setMarkingFunctions(markingFunctions); } diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/ssdeep/SSDeepSimilarityQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/tables/ssdeep/SSDeepSimilarityQueryTransformer.java index ccc22fd1fd5..9fbb4107c3f 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/ssdeep/SSDeepSimilarityQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/ssdeep/SSDeepSimilarityQueryTransformer.java @@ -25,7 +25,7 @@ public class SSDeepSimilarityQueryTransformer extends BaseQueryLogicTransformer< protected final ResponseObjectFactory responseObjectFactory; - public SSDeepSimilarityQueryTransformer(Query query, SSDeepSimilarityQueryConfiguration config, MarkingFunctions markingFunctions, + public SSDeepSimilarityQueryTransformer(Query query, SSDeepSimilarityQueryConfiguration config, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { super(markingFunctions); this.auths = new Authorizations(query.getQueryAuthorizations().split(",")); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/ContentQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/ContentQueryTransformer.java index 0f12a34c2d4..8b41d1e8f1f 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/ContentQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/ContentQueryTransformer.java @@ -34,11 +34,11 @@ public class ContentQueryTransformer extends BaseQueryLogicTransformer metadataIdMap; protected final boolean decodeView; - public ContentQueryTransformer(Query query, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { + public ContentQueryTransformer(Query query, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { this(query, markingFunctions, responseObjectFactory, false); } - public ContentQueryTransformer(Query query, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, boolean decodeView) { + public ContentQueryTransformer(Query query, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, boolean decodeView) { super(markingFunctions); this.auths = new Authorizations(query.getQueryAuthorizations().split(",")); this.responseObjectFactory = responseObjectFactory; diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransform.java b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransform.java index ea9d6fca930..f519573ce4f 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransform.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransform.java @@ -14,7 +14,7 @@ public interface DocumentTransform extends Function,Map.Entry> { // called when adding the document transform - void initialize(Query settings, MarkingFunctions markingFunctions); + void initialize(Query settings, MarkingFunctions markingFunctions); // called after the last document is passed through to get any remaining aggregated results. Map.Entry flush(); @@ -30,11 +30,11 @@ public interface DocumentTransform extends Function,Map. class DefaultDocumentTransform implements DocumentTransform { protected Query settings; - protected MarkingFunctions markingFunctions; + protected MarkingFunctions markingFunctions; protected long queryExecutionForPageStartTime; @Override - public void initialize(Query settings, MarkingFunctions markingFunctions) { + public void initialize(Query settings, MarkingFunctions markingFunctions) { this.settings = settings; this.markingFunctions = markingFunctions; this.queryExecutionForPageStartTime = System.currentTimeMillis(); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java index 79360a8cfa7..0ef05e70c91 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java @@ -50,17 +50,17 @@ public class DocumentTransformer extends DocumentTransformerSupport> logic, Query settings, MarkingFunctions markingFunctions, + public DocumentTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { super(logic, settings, markingFunctions, responseObjectFactory); } - public DocumentTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, + public DocumentTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, Boolean reducedResponse) { super(logic, settings, markingFunctions, responseObjectFactory, reducedResponse); } - public DocumentTransformer(String tableName, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, + public DocumentTransformer(String tableName, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, Boolean reducedResponse) { super(tableName, settings, markingFunctions, responseObjectFactory, reducedResponse); } diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformerSupport.java b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformerSupport.java index 1846db653e5..b3306bcb556 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformerSupport.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformerSupport.java @@ -113,19 +113,19 @@ public abstract class DocumentTransformerSupport extends EventQueryTransfor * @param responseObjectFactory * the response object factory */ - public DocumentTransformerSupport(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, + public DocumentTransformerSupport(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { this(logic, settings, markingFunctions, responseObjectFactory, false); } - public DocumentTransformerSupport(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, + public DocumentTransformerSupport(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, Boolean reducedResponse) { this(null != logic ? logic.getTableName() : null, settings, markingFunctions, responseObjectFactory, reducedResponse); this.logic = logic; } - public DocumentTransformerSupport(String tableName, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, + public DocumentTransformerSupport(String tableName, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, Boolean reducedResponse) { super(tableName, settings, markingFunctions, responseObjectFactory); @@ -188,7 +188,7 @@ protected static Key correctKey(Key origKey) { * @return a collection of the document fields */ protected Collection> buildDocumentFields(Key documentKey, String documentName, Document document, ColumnVisibility topLevelColumnVisibility, - MarkingFunctions markingFunctions) { + MarkingFunctions markingFunctions) { // Whether the fields were added to projectFields or removed from disallowlistedFields, they user does not want them returned // If neither a projection nor a disallowlist was used then the suppressFields set should remain empty @@ -452,7 +452,7 @@ public void writeResultCardinalities() { * @return a collection of fields */ protected Collection> buildDocumentFields(Key documentKey, String fieldName, Attribute attr, ColumnVisibility topLevelColumnVisibility, - MarkingFunctions markingFunctions) { + MarkingFunctions markingFunctions) { Set> myFields = new HashSet<>(); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformer.java index 769cba260e7..55addc5c354 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformer.java @@ -27,7 +27,7 @@ public class EdgeQueryTransformer extends EdgeQueryTransformerSupport,EdgeBase> implements CacheableLogic { private Logger log = Logger.getLogger(EdgeQueryTransformer.class); - public EdgeQueryTransformer(Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, EdgeModelFields fields) { + public EdgeQueryTransformer(Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, EdgeModelFields fields) { super(settings, markingFunctions, responseObjectFactory, fields); } diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformerSupport.java b/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformerSupport.java index e5448a6584c..895bc6fe459 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformerSupport.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformerSupport.java @@ -34,7 +34,7 @@ public abstract class EdgeQueryTransformerSupport extends BaseQueryLogicTra protected ResponseObjectFactory responseObjectFactory; protected EdgeModelFields fields; - public EdgeQueryTransformerSupport(Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, EdgeModelFields fields) { + public EdgeQueryTransformerSupport(Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, EdgeModelFields fields) { super(markingFunctions); this.responseObjectFactory = responseObjectFactory; auths = new Authorizations(settings.getQueryAuthorizations().split(",")); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java index f8f4eeb42f2..f197e1273db 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java @@ -32,11 +32,11 @@ public class EventQueryTransformer extends EventQueryTransformerSupport,EventBase> implements CacheableLogic { - public EventQueryTransformer(String tableName, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { + public EventQueryTransformer(String tableName, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { super(tableName, settings, markingFunctions, responseObjectFactory); } - public EventQueryTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, + public EventQueryTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { super(logic, settings, markingFunctions, responseObjectFactory); } diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformerSupport.java b/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformerSupport.java index 089a5f2f953..f80fcbe10f5 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformerSupport.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformerSupport.java @@ -59,7 +59,7 @@ public abstract class EventQueryTransformerSupport extends BaseQueryLogicTr protected String tableName; protected ResponseObjectFactory responseObjectFactory; - public EventQueryTransformerSupport(String tableName, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { + public EventQueryTransformerSupport(String tableName, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { super(markingFunctions); this.settings = settings; this.auths = new Authorizations(settings.getQueryAuthorizations().split(",")); @@ -67,7 +67,7 @@ public EventQueryTransformerSupport(String tableName, Query settings, MarkingFun this.responseObjectFactory = responseObjectFactory; } - public EventQueryTransformerSupport(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, + public EventQueryTransformerSupport(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { this(logic.getTableName(), settings, markingFunctions, responseObjectFactory); this.logic = logic; diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java index 01f05024134..9821ec80e6e 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java @@ -41,17 +41,17 @@ public class FacetedTransformer extends DocumentTransformerSupport> logic, Query settings, MarkingFunctions markingFunctions, + public FacetedTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { super(logic, settings, markingFunctions, responseObjectFactory); } - public FacetedTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, + public FacetedTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, Boolean reducedResponse) { super(logic, settings, markingFunctions, responseObjectFactory, reducedResponse); } - public FacetedTransformer(String tableName, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, + public FacetedTransformer(String tableName, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, Boolean reducedResponse) { super(tableName, settings, markingFunctions, responseObjectFactory, reducedResponse); } @@ -72,7 +72,7 @@ public FacetedTransformer(String tableName, Query settings, MarkingFunctions mar * @return list of facets */ protected Collection buildFacets(Key documentKey, String fieldName, Document document, ColumnVisibility topLevelColumnVisibility, - MarkingFunctions markingFunctions) { + MarkingFunctions markingFunctions) { Set myFields = new HashSet<>(); @@ -106,7 +106,7 @@ protected Collection buildFacets(Key documentKey, String f * @return list of facets */ protected Collection buildFacets(Key documentKey, String fieldName, Attribute attr, ColumnVisibility topLevelColumnVisibility, - MarkingFunctions markingFunctions) { + MarkingFunctions markingFunctions) { Set myFields = new HashSet<>(); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/FieldIndexCountQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/FieldIndexCountQueryTransformer.java index 456b1138630..823fc635187 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/FieldIndexCountQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/FieldIndexCountQueryTransformer.java @@ -39,7 +39,7 @@ public class FieldIndexCountQueryTransformer extends BaseQueryLogicTransformer> logic = null; private ResponseObjectFactory responseObjectFactory; - public FieldIndexCountQueryTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, + public FieldIndexCountQueryTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { super(markingFunctions); this.auths = new Authorizations(settings.getQueryAuthorizations().split(",")); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/GroupingDocumentTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/GroupingDocumentTransformer.java index 949f3408aaa..1c71976a125 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/GroupingDocumentTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/GroupingDocumentTransformer.java @@ -42,19 +42,19 @@ public class GroupingDocumentTransformer extends DocumentTransformer { private List groupFieldsList; private Map> fieldMap = Maps.newHashMap(); - public GroupingDocumentTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, + public GroupingDocumentTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, Collection groupFieldsSet) { super(logic, settings, markingFunctions, responseObjectFactory); createGroupFieldsList(groupFieldsSet); } - public GroupingDocumentTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, + public GroupingDocumentTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, Collection groupFieldsSet, Boolean reducedResponse) { super(logic, settings, markingFunctions, responseObjectFactory, reducedResponse); createGroupFieldsList(groupFieldsSet); } - public GroupingDocumentTransformer(String tableName, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, + public GroupingDocumentTransformer(String tableName, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, Collection groupFieldsSet, Boolean reducedResponse) { super(tableName, settings, markingFunctions, responseObjectFactory, reducedResponse); createGroupFieldsList(groupFieldsSet); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/GroupingTransform.java b/warehouse/query-core/src/main/java/datawave/query/transformer/GroupingTransform.java index abfb8bd6ba4..d4b21fa52e5 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/GroupingTransform.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/GroupingTransform.java @@ -70,7 +70,7 @@ public class GroupingTransform extends DocumentTransform.DefaultDocumentTransfor * @param markingFunctions * the marking functions */ - public GroupingTransform(GroupFields groupFields, MarkingFunctions markingFunctions, long queryExecutionForPageTimeout) { + public GroupingTransform(GroupFields groupFields, MarkingFunctions markingFunctions, long queryExecutionForPageTimeout) { super.initialize(settings, markingFunctions); this.queryExecutionForPageTimeout = queryExecutionForPageTimeout; this.groups = new Groups(); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/ParentDocumentTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/ParentDocumentTransformer.java index 9b96d420a25..27d5779f66e 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/ParentDocumentTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/ParentDocumentTransformer.java @@ -17,12 +17,12 @@ public class ParentDocumentTransformer extends DocumentTransformer { - public ParentDocumentTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, + public ParentDocumentTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { super(logic, settings, markingFunctions, responseObjectFactory); } - public ParentDocumentTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, + public ParentDocumentTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, Boolean cellLevelVisibility) { super(logic, settings, markingFunctions, responseObjectFactory, cellLevelVisibility); } diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/ShardIndexQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/ShardIndexQueryTransformer.java index c754eb0a348..d6aafa5ae0e 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/ShardIndexQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/ShardIndexQueryTransformer.java @@ -42,7 +42,7 @@ public class ShardIndexQueryTransformer extends BaseQueryLogicTransformer> logic, Query settings, MarkingFunctions markingFunctions, + public ShardIndexQueryTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, QueryModel qm) { super(markingFunctions); this.responseObjectFactory = responseObjectFactory; diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/ShardQueryCountTableTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/ShardQueryCountTableTransformer.java index fad81c8f7d5..a9b63b331ea 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/ShardQueryCountTableTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/ShardQueryCountTableTransformer.java @@ -36,7 +36,7 @@ public class ShardQueryCountTableTransformer extends BaseQueryLogicTransformer variableFieldList = null; private ResponseObjectFactory responseObjectFactory; - public ShardQueryCountTableTransformer(Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { + public ShardQueryCountTableTransformer(Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { super(markingFunctions); this.responseObjectFactory = responseObjectFactory; this.auths = new Authorizations(settings.getQueryAuthorizations().split(",")); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/TagCloudTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/TagCloudTransformer.java index e4847f1dc11..e40c85e8b26 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/TagCloudTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/TagCloudTransformer.java @@ -45,7 +45,7 @@ public enum ResponseVersion { } // TODO-crwill9 pass in state data, not the state object so this is more reusable - public TagCloudTransformer(String responseVersion, Query query, KeywordQueryState state, MarkingFunctions markingFunctions, + public TagCloudTransformer(String responseVersion, Query query, KeywordQueryState state, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, Set> transformers) { super(markingFunctions); this.responseVersion = getResponseVersion(responseVersion); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/TermFrequencyQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/TermFrequencyQueryTransformer.java index ca07a9f3be4..7002e2edab9 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/TermFrequencyQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/TermFrequencyQueryTransformer.java @@ -28,7 +28,7 @@ public class TermFrequencyQueryTransformer extends BaseQueryLogicTransformer markingFunctions, ResponseObjectFactory responseObjectFactory) { super(markingFunctions); this.query = query; this.responseObjectFactory = responseObjectFactory; diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/annotation/AnnotationHitsTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/annotation/AnnotationHitsTransformer.java index 7b2688522b9..a1788e9d3d3 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/annotation/AnnotationHitsTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/annotation/AnnotationHitsTransformer.java @@ -122,7 +122,7 @@ public AnnotationHitsTransformer(ShardQueryConfiguration shardQueryConfig, @Null } @Override - public void initialize(Query settings, MarkingFunctions markingFunctions) { + public void initialize(Query settings, MarkingFunctions markingFunctions) { super.initialize(settings, markingFunctions); // handle query parameters for configuration overrides diff --git a/warehouse/query-core/src/main/java/datawave/query/util/HasMarkingFunctions.java b/warehouse/query-core/src/main/java/datawave/query/util/HasMarkingFunctions.java index 805e5854198..a3f6e37839b 100644 --- a/warehouse/query-core/src/main/java/datawave/query/util/HasMarkingFunctions.java +++ b/warehouse/query-core/src/main/java/datawave/query/util/HasMarkingFunctions.java @@ -4,8 +4,8 @@ public interface HasMarkingFunctions { - void setMarkingFunctions(MarkingFunctions markingFunctions); + void setMarkingFunctions(MarkingFunctions markingFunctions); - MarkingFunctions getMarkingFunctions(); + MarkingFunctions getMarkingFunctions(); } diff --git a/warehouse/query-core/src/test/java/datawave/query/predicate/ValueToAttributesTest.java b/warehouse/query-core/src/test/java/datawave/query/predicate/ValueToAttributesTest.java index dcc4db12af8..01eca42e877 100644 --- a/warehouse/query-core/src/test/java/datawave/query/predicate/ValueToAttributesTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/predicate/ValueToAttributesTest.java @@ -260,7 +260,7 @@ public void testComposites() { TypeMetadata typeMetadata = new TypeMetadata( "dts:[0:beep];types:[0:datawave.data.type.DateType,1:datawave.data.type.IpAddressType,2:datawave.data.type.LcNoDiacriticsType,3:datawave.data.type.NoOpType,4:datawave.data.type.NumberType];MAKE:[0:2];MAKE_COLOR:[0:3];START_DATE:[0:0];TYPE_NOEVAL:[0:2];IP_ADDR:[0:1];WHEELS:[0:2,0:4];COLOR:[0:2];COLOR_WHEELS:[0:3];TYPE:[0:2]"); - MarkingFunctions markingFunctions = new MarkingFunctions.Default(); + MarkingFunctions markingFunctions = new MarkingFunctions.Default(); ValueToAttributes valueToAttributes = new ValueToAttributes(compositeMetadata, typeMetadata, null, markingFunctions, true); } } diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepIngestQueryTest.java b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepIngestQueryTest.java index 1e944d2749b..b8cd3438d0d 100644 --- a/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepIngestQueryTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepIngestQueryTest.java @@ -87,7 +87,7 @@ public static void filterSetup() throws Exception { @Before public void setupQuery() { - MarkingFunctions markingFunctions = new MarkingFunctions.Default(); + MarkingFunctions markingFunctions = new MarkingFunctions.Default(); ResponseObjectFactory responseFactory = new DefaultResponseObjectFactory(); MetadataHelperFactory metadataHelperFactory = new MetadataHelperFactory(); diff --git a/warehouse/query-core/src/test/java/datawave/query/transformer/annotation/AnnotationHitsTransformerTest.java b/warehouse/query-core/src/test/java/datawave/query/transformer/annotation/AnnotationHitsTransformerTest.java index 2a2a88c152b..5ee5b2134b4 100644 --- a/warehouse/query-core/src/test/java/datawave/query/transformer/annotation/AnnotationHitsTransformerTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/transformer/annotation/AnnotationHitsTransformerTest.java @@ -117,7 +117,7 @@ public class AnnotationHitsTransformerTest { private ShardQueryConfiguration shardQueryConfiguration; private Query settings; - private MarkingFunctions markingFunctions; + private MarkingFunctions markingFunctions; private List annotations; private AnnotationSource annotationSource; diff --git a/web-services/metrics/src/main/java/datawave/query/metrics/ShardTableQueryMetricHandler.java b/web-services/metrics/src/main/java/datawave/query/metrics/ShardTableQueryMetricHandler.java index 7510bee7c2f..304cac6efa4 100644 --- a/web-services/metrics/src/main/java/datawave/query/metrics/ShardTableQueryMetricHandler.java +++ b/web-services/metrics/src/main/java/datawave/query/metrics/ShardTableQueryMetricHandler.java @@ -24,6 +24,8 @@ import javax.enterprise.context.ApplicationScoped; import javax.inject.Inject; +import datawave.core.common.util.AccessExpressionUtil; +import datawave.marking.Markings; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; @@ -277,14 +279,12 @@ private Multimap getEntries(AbstractColumnBasedHandler event.setDataType(type); event.setTimestamp(storedQueryMetric.getCreateDate().getTime()); // get security markings from metric, otherwise default to PUBLIC - Map markings = updatedQueryMetric.getMarkings(); + Markings markings = updatedQueryMetric.getMarkings(); if (markings == null || markings.isEmpty()) { event.setVisibility(new ColumnVisibility(DEFAULT_SECURITY_MARKING)); } else { try { - // Extract the visibility expression from the legacy Map markings - String visExpr = markings.getOrDefault("columnVisibility", DEFAULT_SECURITY_MARKING); - event.setVisibility(new ColumnVisibility(visExpr)); + event.setVisibility(AccessExpressionUtil.toColumnVisibility(markings)); } catch (Exception e) { log.error(e.getMessage(), e); event.setVisibility(new ColumnVisibility(DEFAULT_SECURITY_MARKING)); diff --git a/web-services/query/src/test/java/datawave/webservice/query/logic/TestLegacyBaseQueryLogicTransformer.java b/web-services/query/src/test/java/datawave/webservice/query/logic/TestLegacyBaseQueryLogicTransformer.java index 57bf97e9678..a640dcb7ce9 100644 --- a/web-services/query/src/test/java/datawave/webservice/query/logic/TestLegacyBaseQueryLogicTransformer.java +++ b/web-services/query/src/test/java/datawave/webservice/query/logic/TestLegacyBaseQueryLogicTransformer.java @@ -75,7 +75,7 @@ public void testCreateResponse_ResultsPageComplete() { private static class TestTransformer extends BaseQueryLogicTransformer,EventBase> { BaseQueryResponse response; - public TestTransformer(MarkingFunctions markingFunctions, BaseQueryResponse response) { + public TestTransformer(MarkingFunctions markingFunctions, BaseQueryResponse response) { super(markingFunctions); this.response = response; } diff --git a/web-services/query/src/test/java/datawave/webservice/query/logic/composite/CompositeQueryLogicTest.java b/web-services/query/src/test/java/datawave/webservice/query/logic/composite/CompositeQueryLogicTest.java index 372d0f17322..5aaa959e554 100644 --- a/web-services/query/src/test/java/datawave/webservice/query/logic/composite/CompositeQueryLogicTest.java +++ b/web-services/query/src/test/java/datawave/webservice/query/logic/composite/CompositeQueryLogicTest.java @@ -185,7 +185,7 @@ public void addResponse(TestQueryResponse response) { public static class TestQueryLogicTransformer extends BaseQueryLogicTransformer,TestQueryResponse> { - public TestQueryLogicTransformer(MarkingFunctions markingFunctions) { + public TestQueryLogicTransformer(MarkingFunctions markingFunctions) { super(markingFunctions); } @@ -225,7 +225,7 @@ public BaseQueryResponse createResponse(List resultList) { public static class DifferentTestQueryLogicTransformer extends BaseQueryLogicTransformer,TestQueryResponse> { - public DifferentTestQueryLogicTransformer(MarkingFunctions markingFunctions) { + public DifferentTestQueryLogicTransformer(MarkingFunctions markingFunctions) { super(markingFunctions); } From 76f2cbb4bc939df232ffccc5a367a6ad404a320b Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Fri, 20 Mar 2026 13:53:18 -0400 Subject: [PATCH 04/63] fixes --- .mvn/maven-build-cache-config.xml | 2 +- .../cachedresults/CacheableQueryRowImpl.java | 20 ++-- .../common/util/AccessExpressionUtil.java | 5 +- .../modification/MutableMetadataHandler.java | 3 +- .../datawave/marking/MarkingFunctions.java | 16 +++- .../MarkingFunctionsDefaultCombineTest.java | 8 +- .../edge/DefaultEdgeDictionary.java | 8 +- .../metadata/DefaultMetadataFieldScanner.java | 3 +- .../querymetric/QueryMetricTest.java | 9 +- .../querymetric/QueryMetricOperations.java | 2 +- .../handler/ShardTableQueryMetricHandler.java | 4 +- .../querymetric/QueryMetricTestBase.java | 6 +- .../query/AbstractQueryServiceTest.java | 2 +- .../data/config/ingest/CompositeIngest.java | 7 +- .../data/config/ingest/VirtualIngest.java | 7 +- .../handler/edge/define/EdgeDataBundle.java | 92 ++++++++++--------- .../csv/NormalizedContentInterfaceTest.java | 2 - .../util/keyword/DefaultTagCloudUtils.java | 20 ++-- .../java/datawave/util/keyword/TagCloud.java | 24 +++-- .../datawave/util/keyword/TagCloudUtils.java | 8 +- .../query/predicate/ValueToAttributes.java | 9 +- .../parser/TermFrequencyKeyValueFactory.java | 3 +- .../shard/FieldIndexCountQueryLogic.java | 9 +- .../transformer/DocumentTransformer.java | 4 +- .../EdgeQueryTransformerSupport.java | 7 +- .../query/transformer/FacetedTransformer.java | 2 +- .../transformer/TagCloudTransformer.java | 17 ++-- .../query/tables/RemoteEventQueryLogicIT.java | 2 +- .../metrics/ShardTableQueryMetricHandler.java | 4 +- 29 files changed, 161 insertions(+), 144 deletions(-) diff --git a/.mvn/maven-build-cache-config.xml b/.mvn/maven-build-cache-config.xml index 420d20c83df..1e22cfc0381 100644 --- a/.mvn/maven-build-cache-config.xml +++ b/.mvn/maven-build-cache-config.xml @@ -23,7 +23,7 @@ --> - false + true SHA-256 true diff --git a/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java b/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java index 6e04edd6e4d..cba6612aafa 100644 --- a/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java +++ b/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java @@ -24,10 +24,9 @@ import datawave.webservice.query.data.ObjectSizeOf; import datawave.webservice.query.util.TypedValue; -@SuppressWarnings("unchecked") public class CacheableQueryRowImpl extends CacheableQueryRow implements ObjectSizeOf { - private static Logger log = LoggerFactory.getLogger(CacheableQueryRowImpl.class); + private static final Logger LOGGER = LoggerFactory.getLogger(CacheableQueryRowImpl.class); private static final Markings EMPTY_MARKINGS = AccessExpressionMarkings.builder().accessExpression(MarkingFunctions.Default.ACCESS.newExpression("")) .build(); @@ -68,20 +67,17 @@ public void addColumn(String columnName, TypedValue columnTypedValue, Markings expressions = new HashSet<>(); - expressions.add(markingFunctions.toAccessExpression(this.markings)); - expressions.add(markingFunctions.toAccessExpression(markings)); - AccessExpression combined = markingFunctions.combine(expressions); - + AccessExpression combined = markingFunctions + .combine(List.of(markingFunctions.toAccessExpression(this.markings), markingFunctions.toAccessExpression(markings))); // use combined marking as new markings this.markings = markingFunctions.fromAccessExpression(combined); } catch (MarkingFunctions.Exception e) { - log.error("Invalid markings {} skipping column {} = {}", markings, columnName, columnTypedValue, e); + LOGGER.error("Invalid markings {} skipping column {} = {}", markings, columnName, columnTypedValue, e); return; } } @@ -139,7 +135,7 @@ private void manageColumnInsert(Type datawaveType, String columnName, TypedVa if (!currMarkings.equals(markings)) { try { - Set expressions = new HashSet<>(); + List expressions = new ArrayList<>(); expressions.add(markingFunctions.toAccessExpression(currMarkings)); expressions.add(markingFunctions.toAccessExpression(markings)); AccessExpression combined = markingFunctions.combine(expressions); @@ -293,7 +289,7 @@ public void setMarkings(Markings markings) { markingFunctions.toAccessExpression(markings); this.markings = markings; } catch (MarkingFunctions.Exception e) { - log.error("Invalid markings {}", markings, e); + LOGGER.error("Invalid markings {}", markings, e); } } @@ -322,7 +318,7 @@ public String getColumnSecurityMarkingString(Map columnMap) { try { mStr = markingFunctions.toAccessExpression(m).getExpression(); } catch (MarkingFunctions.Exception e) { - log.error("could not serialize markings {}", m, e); + LOGGER.error("could not serialize markings {}", m, e); } } if (v == null) { diff --git a/core/common-util/src/main/java/datawave/core/common/util/AccessExpressionUtil.java b/core/common-util/src/main/java/datawave/core/common/util/AccessExpressionUtil.java index 379eceabb39..d0c9bab2c1f 100644 --- a/core/common-util/src/main/java/datawave/core/common/util/AccessExpressionUtil.java +++ b/core/common-util/src/main/java/datawave/core/common/util/AccessExpressionUtil.java @@ -2,12 +2,13 @@ import static java.nio.charset.StandardCharsets.UTF_8; -import datawave.marking.AccessExpressionMarkings; -import datawave.marking.Markings; import org.apache.accumulo.access.Access; import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.security.ColumnVisibility; +import datawave.marking.AccessExpressionMarkings; +import datawave.marking.Markings; + /** * Utility for bridging between Accumulo's {@link ColumnVisibility} (from accumulo-core) and {@link AccessExpression} (from accumulo-access). *

diff --git a/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java b/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java index 9326aaceddc..5e87c47f8c2 100644 --- a/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java +++ b/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java @@ -374,8 +374,7 @@ public void process(AccumuloClient client, ModificationRequestBase request, Map< // Count the history entries if history is going to be inserted. if (insertHistory && (MODE.INSERT.equals(mode) || MODE.UPDATE.equals(mode))) { - List> fieldHistoryList = getField(client, userAuths, shardId, datatype, eventUid, "HISTORY_" + fieldName, null, null, - null); + List> fieldHistoryList = getField(client, userAuths, shardId, datatype, eventUid, "HISTORY_" + fieldName, null, null, null); for (Pair p : fieldHistoryList) { if (p.getLeft().getColumnQualifier().find(mr.getFieldValue()) > -1) { diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java index 024579e86f9..cbe8a8d39b7 100644 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java @@ -2,6 +2,8 @@ import java.util.Arrays; import java.util.Collection; +import java.util.HashSet; +import java.util.Set; import java.util.stream.Collectors; import org.apache.accumulo.access.Access; @@ -71,12 +73,24 @@ public Exception(Throwable cause) { class Default implements MarkingFunctions { public static final Access ACCESS = Access.builder().build(); + public static final AccessExpression EMPTY_MARKINGS = ACCESS.newExpression(""); @Override public AccessExpression combine(Collection expressions) { // filter out any empty expressions and concatenate with '&' // @formatter:off - ParsedAccessExpression expression = ACCESS.newParsedExpression(expressions + + Set uniqueExpressions = new HashSet<>(expressions); + + if (uniqueExpressions.isEmpty()) { + return EMPTY_MARKINGS; + } + + if (uniqueExpressions.size() == 1) { + return uniqueExpressions.stream().findFirst().get(); + } + + ParsedAccessExpression expression = ACCESS.newParsedExpression(uniqueExpressions .stream() .map(AccessExpression::getExpression) .filter(str -> !str.isEmpty()) diff --git a/core/utils/accumulo-utils/src/test/java/datawave/marking/MarkingFunctionsDefaultCombineTest.java b/core/utils/accumulo-utils/src/test/java/datawave/marking/MarkingFunctionsDefaultCombineTest.java index f9867efa87c..6cf06e763fe 100644 --- a/core/utils/accumulo-utils/src/test/java/datawave/marking/MarkingFunctionsDefaultCombineTest.java +++ b/core/utils/accumulo-utils/src/test/java/datawave/marking/MarkingFunctionsDefaultCombineTest.java @@ -2,12 +2,12 @@ import static org.junit.jupiter.api.Assertions.assertEquals; +import java.util.List; + import org.apache.accumulo.access.Access; import org.apache.accumulo.access.AccessExpression; import org.junit.jupiter.api.Test; -import com.google.common.collect.Sets; - public class MarkingFunctionsDefaultCombineTest { public static final Access ACCESS = Access.builder().build(); @@ -18,7 +18,7 @@ public void testCombineAnds() throws MarkingFunctions.Exception { AccessExpression oneAnna = ACCESS.newExpression("A&B"); AccessExpression twoAnna = ACCESS.newExpression("A&C"); - AccessExpression combined = markingFunctions.combine(Sets.newHashSet(oneAnna, twoAnna)); + AccessExpression combined = markingFunctions.combine(List.of(oneAnna, twoAnna)); assertEquals(ACCESS.newExpression("A&B&C"), combined); } @@ -29,7 +29,7 @@ public void testCombineOrs() throws MarkingFunctions.Exception { AccessExpression oneOr = ACCESS.newExpression("A|B"); AccessExpression twoOr = ACCESS.newExpression("A|C"); - AccessExpression combined = markingFunctions.combine(Sets.newHashSet(oneOr, twoOr)); + AccessExpression combined = markingFunctions.combine(List.of(oneOr, twoOr)); assertEquals(ACCESS.newExpression("(A|B)&(A|C)"), combined); } diff --git a/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/edge/DefaultEdgeDictionary.java b/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/edge/DefaultEdgeDictionary.java index db34844f810..4a4f91c80dd 100644 --- a/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/edge/DefaultEdgeDictionary.java +++ b/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/edge/DefaultEdgeDictionary.java @@ -24,6 +24,7 @@ import io.protostuff.Message; import io.protostuff.Output; import io.protostuff.Schema; +import lombok.Setter; @XmlRootElement(name = "EdgeDictionary") @XmlAccessorType(XmlAccessType.NONE) @@ -33,6 +34,7 @@ public class DefaultEdgeDictionary extends EdgeDictionaryBase getSchema() { return SCHEMA; } @@ -227,7 +225,7 @@ public String getMainContent() { fieldBuilder.append(field).append(SEP); } - String fieldNames = fieldBuilder.toString().substring(0, fieldBuilder.length() - 2); + String fieldNames = fieldBuilder.substring(0, fieldBuilder.length() - 2); String date = metadata.getStartDate(); builder.append("").append(type).append(""); diff --git a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java index ff525c6b4ee..68dc223f7ef 100644 --- a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java +++ b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java @@ -275,7 +275,8 @@ private void setFieldNameAndAlias() { private void setDescriptions() throws MarkingFunctions.Exception { DefaultDescription description = responseObjectFactory.getDescription(); description.setDescription(currValue.toString()); - description.setMarkings(markingFunctions.fromAccessExpression(MarkingFunctions.Default.ACCESS.newExpression(currKey.getColumnVisibility().toString()))); + description.setMarkings( + markingFunctions.fromAccessExpression(MarkingFunctions.Default.ACCESS.newExpression(currKey.getColumnVisibility().toString()))); currField.getDescriptions().add(description); } diff --git a/microservices/services/query-metric/api/src/test/java/datawave/microservice/querymetric/QueryMetricTest.java b/microservices/services/query-metric/api/src/test/java/datawave/microservice/querymetric/QueryMetricTest.java index 1a7d289fd07..d078b0979da 100644 --- a/microservices/services/query-metric/api/src/test/java/datawave/microservice/querymetric/QueryMetricTest.java +++ b/microservices/services/query-metric/api/src/test/java/datawave/microservice/querymetric/QueryMetricTest.java @@ -51,8 +51,7 @@ public class QueryMetricTest { @BeforeAll public static void setup() { queryMetric = new QueryMetric(); - queryMetric.setMarkings(AccessExpressionMarkings.builder() - .accessExpression(MarkingFunctions.Default.ACCESS.newExpression("PUBLIC")).build()); + queryMetric.setMarkings(AccessExpressionMarkings.builder().accessExpression(MarkingFunctions.Default.ACCESS.newExpression("PUBLIC")).build()); negativeSelectors = new ArrayList<>(); negativeSelectors.add("negativeSelector1"); positiveSelectors = new ArrayList<>(); @@ -101,8 +100,7 @@ public void testSettersGetters() { queryMetric.setHost("host"); queryMetric.setLastUpdated(d); queryMetric.setLifecycle(Lifecycle.INITIALIZED); - queryMetric.setMarkings(AccessExpressionMarkings.builder() - .accessExpression(MarkingFunctions.Default.ACCESS.newExpression("PUBLIC")).build()); + queryMetric.setMarkings(AccessExpressionMarkings.builder().accessExpression(MarkingFunctions.Default.ACCESS.newExpression("PUBLIC")).build()); queryMetric.setNegativeSelectors(negativeSelectors); queryMetric.setNumUpdates(0); queryMetric.setPageTimes(pageTimes); @@ -232,8 +230,7 @@ public void testVersionSerialization() throws Exception { qm.setHost("host"); qm.setLastUpdated(d); qm.setLifecycle(BaseQueryMetric.Lifecycle.INITIALIZED); - qm.setMarkings(AccessExpressionMarkings.builder() - .accessExpression(MarkingFunctions.Default.ACCESS.newExpression("PUBLIC")).build()); + qm.setMarkings(AccessExpressionMarkings.builder().accessExpression(MarkingFunctions.Default.ACCESS.newExpression("PUBLIC")).build()); qm.setNegativeSelectors(negativeSelectors); qm.setNumUpdates(0); qm.setPageTimes(pageTimes); diff --git a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java index 2aa2f0526c3..d0c82dcd3b5 100644 --- a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java +++ b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java @@ -25,7 +25,6 @@ import javax.annotation.PreDestroy; import javax.annotation.security.PermitAll; -import datawave.core.common.util.AccessExpressionUtil; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.accumulo.core.security.VisibilityEvaluator; @@ -54,6 +53,7 @@ import com.hazelcast.map.IMap; import com.hazelcast.spring.cache.HazelcastCacheManager; +import datawave.core.common.util.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.microservice.authorization.user.DatawaveUserDetails; import datawave.microservice.querymetric.config.QueryMetricTransportType; diff --git a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java index f877a9a9cb1..7144b764567 100644 --- a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java +++ b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java @@ -19,8 +19,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantReadWriteLock; -import datawave.core.common.util.AccessExpressionUtil; -import datawave.marking.Markings; import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.AccumuloException; @@ -51,6 +49,7 @@ import com.google.common.collect.Multimap; import datawave.core.common.connection.AccumuloConnectionFactory; +import datawave.core.common.util.AccessExpressionUtil; import datawave.core.query.util.QueryUtil; import datawave.data.hash.UID; import datawave.data.hash.UIDBuilder; @@ -66,6 +65,7 @@ import datawave.ingest.protobuf.Uid; import datawave.ingest.table.config.TableConfigHelper; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.microservice.authorization.user.DatawaveUserDetails; import datawave.microservice.query.Query; import datawave.microservice.query.QueryImpl; diff --git a/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java b/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java index d65064d0c9b..e6d2f7ea944 100644 --- a/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java +++ b/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java @@ -19,9 +19,6 @@ import java.util.Map; import java.util.Set; -import datawave.marking.AccessExpressionMarkings; -import datawave.marking.MarkingFunctions; -import datawave.marking.Markings; import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.BatchScanner; @@ -69,7 +66,10 @@ import datawave.core.common.connection.AccumuloConnectionFactory; import datawave.ingest.protobuf.Uid; +import datawave.marking.AccessExpressionMarkings; import datawave.marking.ColumnVisibilitySecurityMarking; +import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.microservice.authorization.preauth.ProxiedEntityX509Filter; import datawave.microservice.authorization.user.DatawaveUserDetails; import datawave.microservice.querymetric.config.QueryMetricClientProperties; diff --git a/microservices/services/query/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java b/microservices/services/query/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java index 9aa72368f5d..1ddb7ad41bd 100644 --- a/microservices/services/query/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java +++ b/microservices/services/query/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java @@ -27,7 +27,6 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import datawave.marking.MarkingFunctions; import org.apache.http.NameValuePair; import org.apache.http.client.utils.URLEncodedUtils; import org.junit.jupiter.api.AfterEach; @@ -72,6 +71,7 @@ import com.hazelcast.core.HazelcastInstance; import datawave.marking.ColumnVisibilitySecurityMarking; +import datawave.marking.MarkingFunctions; import datawave.microservice.audit.AuditClient; import datawave.microservice.authorization.jwt.JWTRestTemplate; import datawave.microservice.authorization.user.DatawaveUserDetails; diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CompositeIngest.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CompositeIngest.java index bfd12d9e412..62056a598b8 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CompositeIngest.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CompositeIngest.java @@ -14,7 +14,6 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; -import datawave.marking.AccessExpressionMarkings; import org.apache.commons.lang.NotImplementedException; import org.apache.hadoop.conf.Configuration; import org.slf4j.Logger; @@ -30,6 +29,7 @@ import datawave.ingest.data.config.GroupedNormalizedContentInterface; import datawave.ingest.data.config.NormalizedContentInterface; import datawave.ingest.data.config.NormalizedFieldAndValue; +import datawave.marking.AccessExpressionMarkings; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; @@ -305,9 +305,10 @@ protected Markings mergeMarkings(Markings markings1, Markings markings2 if (markings1 instanceof AccessExpressionMarkings && markings2 instanceof AccessExpressionMarkings) { AccessExpressionMarkings aem1 = (AccessExpressionMarkings) markings1; AccessExpressionMarkings aem2 = (AccessExpressionMarkings) markings2; - return markingFunctions.fromAccessExpression(markingFunctions.combine(Set.of(aem1.getMarkings(), aem2.getMarkings()))); + return markingFunctions.fromAccessExpression(markingFunctions.combine(List.of(aem1.getMarkings(), aem2.getMarkings()))); } else { - throw new RuntimeException(String.format("Unknown markings class %s or %s", markings1.getClass().getName(), markings1.getClass().getName())); + throw new RuntimeException( + String.format("Unknown markings class %s or %s", markings1.getClass().getName(), markings1.getClass().getName())); } } catch (MarkingFunctions.Exception e) { throw new RuntimeException("Unable to combine markings.", e); diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java index 6bec75acdef..eee3e6ae3bd 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java @@ -13,7 +13,6 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import datawave.marking.AccessExpressionMarkings; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.builder.HashCodeBuilder; import org.apache.commons.lang3.builder.EqualsBuilder; @@ -28,6 +27,7 @@ import datawave.ingest.data.config.GroupedNormalizedContentInterface; import datawave.ingest.data.config.NormalizedContentInterface; import datawave.ingest.data.config.NormalizedFieldAndValue; +import datawave.marking.AccessExpressionMarkings; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; @@ -358,9 +358,10 @@ protected Markings mergeMarkings(Markings markings1, Markings markings2 if (markings1 instanceof AccessExpressionMarkings && markings2 instanceof AccessExpressionMarkings) { AccessExpressionMarkings aem1 = (AccessExpressionMarkings) markings1; AccessExpressionMarkings aem2 = (AccessExpressionMarkings) markings2; - return markingFunctions.fromAccessExpression(markingFunctions.combine(Set.of(aem1.getMarkings(), aem2.getMarkings()))); + return markingFunctions.fromAccessExpression(markingFunctions.combine(List.of(aem1.getMarkings(), aem2.getMarkings()))); } else { - throw new RuntimeException(String.format("Unknown markings class %s or %s", markings1.getClass().getName(), markings1.getClass().getName())); + throw new RuntimeException( + String.format("Unknown markings class %s or %s", markings1.getClass().getName(), markings1.getClass().getName())); } } catch (MarkingFunctions.Exception e) { throw new RuntimeException("Unable to combine markings.", e); diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/EdgeDataBundle.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/EdgeDataBundle.java index d4659e72bad..413569db5a7 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/EdgeDataBundle.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/EdgeDataBundle.java @@ -18,8 +18,8 @@ import datawave.ingest.data.config.NormalizedContentInterface; import datawave.ingest.data.config.ingest.IngestHelperInterface; import datawave.ingest.mapreduce.handler.edge.define.VertexValue.ValueType; +import datawave.marking.AccessExpressionMarkings; import datawave.marking.MarkingFunctions; -import datawave.marking.MarkingFunctions.Exception; import datawave.marking.Markings; import datawave.util.time.DateHelper; @@ -34,43 +34,43 @@ public class EdgeDataBundle { // Input/Setup variables // final so you're not tempted to change them private final RawRecordContainer event; + private final MarkingFunctions markingFunctions; private VertexValue source; private VertexValue sink; - private EdgeDefinition edgeDefinition = null; + private EdgeDefinition edgeDefinition; private EdgeDirection edgeDirection = EdgeDirection.UNIDIRECTIONAL; - private String sourceMaskedValue = null; - private boolean hasMaskedSource = false; - private String sinkMaskedValue = null; - private boolean hasMaskedSink = false; + private String sourceMaskedValue; + private boolean hasMaskedSource; + private String sinkMaskedValue; + private boolean hasMaskedSink; // Variables for output - private boolean requiresMasking = false; - private boolean isDeleting = false; + private boolean requiresMasking; + private boolean isDeleting; private String edgeType; private String enrichedIndex; private String enrichedValue; private String edgeAttribute3; private String edgeAttribute2; - private long eventDate = 0; + private long eventDate; private IngestHelperInterface helper; - private Markings markings = null; - private ColumnVisibility maskedVisibility = null; - private boolean forceMaskedVisibility = false; // if this is a masked event, but the + private Markings markings; + private ColumnVisibility maskedVisibility; + private boolean forceMaskedVisibility; // if this is a masked event, but the // the fields defined aren't masked, use the unmasked visibility // Duration value if defined - private DurationValue durationValue = null; + private DurationValue durationValue; private String loadDate; - private MarkingFunctions mf = null; private String uuid; private long activityDate; private boolean validActivityDate; public EdgeDataBundle(RawRecordContainer event, String typeName, String id, IngestHelperInterface helper) { - this.mf = MarkingFunctions.Factory.createMarkingFunctions(); + this.markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); this.event = event; this.eventDate = event.getDate(); this.edgeType = typeName; @@ -80,7 +80,7 @@ public EdgeDataBundle(RawRecordContainer event, String typeName, String id, Inge public EdgeDataBundle(EdgeDefinition edgeDef, NormalizedContentInterface ifaceSource, NormalizedContentInterface ifaceSink, RawRecordContainer event, IngestHelperInterface helper) { - this(event, edgeDef.getEdgeType().toString(), null, helper); + this(event, edgeDef.getEdgeType(), null, helper); this.setSource(new VertexValue(edgeDef.isUseRealm(), edgeDef.getSourceIndexedFieldRealm(), edgeDef.getSourceEventFieldRealm(), edgeDef.getSourceRelationship(), edgeDef.getSourceCollection(), ifaceSource)); @@ -140,20 +140,34 @@ public void setMaskedVisibility(ColumnVisibility maskedVisibility) { this.maskedVisibility = maskedVisibility; } - @SuppressWarnings("unchecked") - private void initMarkings(Markings m1, Markings m2) { - if (m1 != null) { - if (m2 != null) { + /** + * A helper routine to merge markings when merging fields of a NormalizedContentInterface + * + * @param markings1 + * markings + * @param markings2 + * different markings + */ + protected void initMarkings(Markings markings1, Markings markings2) { + if (markings2 != null) { + if (markings1 == null) { + this.markings = markings2; + } else { try { - this.markings = mf.combine(m1, m2); - } catch (Exception e) { - throw new RuntimeException("Unable to combine markings", e); + if (markings1 instanceof AccessExpressionMarkings && markings2 instanceof AccessExpressionMarkings) { + AccessExpressionMarkings aem1 = (AccessExpressionMarkings) markings1; + AccessExpressionMarkings aem2 = (AccessExpressionMarkings) markings2; + this.markings = markingFunctions.fromAccessExpression(markingFunctions.combine(List.of(aem1.getMarkings(), aem2.getMarkings()))); + } else { + throw new RuntimeException( + String.format("Unknown markings class %s or %s", markings1.getClass().getName(), markings1.getClass().getName())); + } + } catch (MarkingFunctions.Exception e) { + throw new RuntimeException("Unable to combine markings.", e); } - } else { - this.markings = m1; } - } else if (m2 != null) { - this.markings = m2; + } else { + this.markings = markings1; } } @@ -192,24 +206,19 @@ public Value getEdgeValue(boolean forwardEdge, EdgeKey.DATE_TYPE date_type) { } if (date_type == EdgeKey.DATE_TYPE.EVENT_ONLY) { - if (validActivityDate) { - builder.setBadActivityDate(false); - } else { - builder.setBadActivityDate(true); - } - + builder.setBadActivityDate(!validActivityDate); } // Set counts if (!this.isDeleting()) { - builder.setCount(1l); + builder.setCount(1L); } else { - builder.setCount(-1l); + builder.setCount(-1L); } // Set Hour Bitmask if (hour != -1) { builder.setHour(hour); } - if (forwardEdge == true) { + if (forwardEdge) { builder.setSourceValue(source.getValue(ValueType.EVENT)); builder.setSinkValue(sink.getValue(ValueType.EVENT)); } else { @@ -236,16 +245,11 @@ public Value getStatsActivityValue(boolean forwardEdge, EdgeKey.DATE_TYPE date_t } if (date_type == EdgeKey.DATE_TYPE.EVENT_ONLY) { - if (validActivityDate) { - builder.setBadActivityDate(false); - } else { - builder.setBadActivityDate(true); - } - + builder.setBadActivityDate(!validActivityDate); } List hours = EdgeValueHelper.getLongListForHour(hour, this.isDeleting()); builder.setHours(hours); - if (forwardEdge == true) { + if (forwardEdge) { builder.setSourceValue(source.getValue(ValueType.EVENT)); } else { builder.setSourceValue(sink.getValue(ValueType.EVENT)); @@ -276,7 +280,7 @@ public Value getDurationAsValue(boolean forwardEdge) { EdgeValueBuilder builder = datawave.edge.util.EdgeValue.newBuilder(); List duration = EdgeValueHelper.getLongListForDuration(this.getDuration(), this.isDeleting()); builder.setDuration(duration); - if (forwardEdge == true) { + if (forwardEdge) { builder.setSourceValue(source.getValue(ValueType.EVENT)); } else { builder.setSourceValue(sink.getValue(ValueType.EVENT)); diff --git a/warehouse/ingest-csv/src/test/java/datawave/ingest/csv/NormalizedContentInterfaceTest.java b/warehouse/ingest-csv/src/test/java/datawave/ingest/csv/NormalizedContentInterfaceTest.java index ca875cbf8be..035e4dcb295 100644 --- a/warehouse/ingest-csv/src/test/java/datawave/ingest/csv/NormalizedContentInterfaceTest.java +++ b/warehouse/ingest-csv/src/test/java/datawave/ingest/csv/NormalizedContentInterfaceTest.java @@ -1,8 +1,6 @@ package datawave.ingest.csv; import java.io.IOException; -import java.util.HashMap; -import java.util.Map; import java.util.Objects; import org.apache.hadoop.conf.Configuration; diff --git a/warehouse/keyword-common/src/main/java/datawave/util/keyword/DefaultTagCloudUtils.java b/warehouse/keyword-common/src/main/java/datawave/util/keyword/DefaultTagCloudUtils.java index 3ad1b52735f..f6983c178e5 100644 --- a/warehouse/keyword-common/src/main/java/datawave/util/keyword/DefaultTagCloudUtils.java +++ b/warehouse/keyword-common/src/main/java/datawave/util/keyword/DefaultTagCloudUtils.java @@ -2,7 +2,6 @@ import java.io.Serializable; import java.util.Collection; -import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.Map; @@ -10,9 +9,11 @@ import java.util.Set; import java.util.stream.Collectors; -import org.apache.accumulo.core.security.ColumnVisibility; +import org.apache.accumulo.access.AccessExpression; import org.apache.commons.lang3.StringUtils; +import datawave.marking.MarkingFunctions; + /** * Default implementations for pluggable utilities for generating tag clouds, includes mechanisms to partition keywords into separate tag clouds, combine or * merge visibility strings, and calculate scores, source collections and frequencies of individual keywords based on observed results @@ -22,17 +23,12 @@ public class DefaultTagCloudUtils implements TagCloudUtils, Serializable { private static final String MULTI_VALUE_SEPARATOR = ","; @Override - public Map generateCombinedVisibility(Set visibilities) { - final StringBuilder b = new StringBuilder(); - visibilities.forEach(x -> b.append("(").append(x).append(")&")); - if (b.length() > 0) { - b.setLength(b.length() - 1); - ColumnVisibility cv = new ColumnVisibility(b.toString()); - ColumnVisibility flat = new ColumnVisibility(cv.flatten()); - return Map.of("visibility", flat.toString()); - } else { - return Collections.emptyMap(); + public AccessExpression generateCombinedVisibility(Set visibilities) { + if (visibilities == null || visibilities.isEmpty()) { + return null; } + Set expressions = visibilities.stream().map(MarkingFunctions.Default.ACCESS::newExpression).collect(Collectors.toSet()); + return new MarkingFunctions.Default().combine(expressions); } @Override diff --git a/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloud.java b/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloud.java index 584803325f9..8c338f2b7e0 100644 --- a/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloud.java +++ b/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloud.java @@ -12,8 +12,12 @@ import java.util.SortedSet; import java.util.TreeSet; +import org.apache.accumulo.access.AccessExpression; + import com.google.gson.Gson; +import datawave.marking.MarkingFunctions; + /** A tag cloud - a collection of tags that have a keyword, score, frequency and list of sources from which they originated */ public class TagCloud { @@ -22,8 +26,8 @@ public class TagCloud { /** metadata for the cloud */ final Map metadata; - /** the 'visibility' of this cloud */ - final Map visibility; + /** the combined visibility expression string for this cloud (stored as String for Gson serialization) */ + final String visibility; /** the sorted set of keywords in this cloud, including scores and sources */ final SortedSet results; @@ -34,13 +38,13 @@ public class TagCloud { * @param metadata * metadata for the tag cloud * @param visibility - * the visibility of the cloud, some form of visibility marking + * the combined access expression for the cloud, or {@code null} * @param results * the entries that belong in the tag cloud. */ - protected TagCloud(Map metadata, Map visibility, SortedSet results) { + protected TagCloud(Map metadata, AccessExpression visibility, SortedSet results) { this.metadata = metadata; - this.visibility = visibility; + this.visibility = (visibility != null) ? visibility.getExpression() : ""; this.results = results; } @@ -48,8 +52,12 @@ public Map getMetadata() { return metadata; } - public Map getVisibility() { - return visibility; + /** Return the combined visibility as an {@link AccessExpression}. */ + public AccessExpression getVisibility() { + if (visibility == null || visibility.isEmpty()) { + return null; + } + return MarkingFunctions.Default.ACCESS.newExpression(visibility); } public Collection getResults() { @@ -213,7 +221,7 @@ public List build() { final SortedSet results = new TreeSet<>(comparator); results.addAll(e.getValue()); String partition = getPartition(e.getKey()); - Map visibility = utils.generateCombinedVisibility(visibilities.get(partition)); + AccessExpression visibility = utils.generateCombinedVisibility(visibilities.get(partition)); Map tagCloudMetadata = utils.generateCombinedMetadata(this.metadata.get(partition)); tagClouds.add(new TagCloud(tagCloudMetadata, visibility, results)); } diff --git a/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloudUtils.java b/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloudUtils.java index ecfb1f80d39..23fcb5d4c23 100644 --- a/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloudUtils.java +++ b/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloudUtils.java @@ -5,19 +5,21 @@ import java.util.Map; import java.util.Set; +import org.apache.accumulo.access.AccessExpression; + /** * Interface for pluggable utilities for generating tag clouds, includes mechanisms to partition keywords into separate tag clouds, combine or merge visibility * strings, and calculate scores, source collections and frequencies of individual keywords based on observed results */ public interface TagCloudUtils { /** - * Aggregate visibilities by concatenating them together with an AND ('&'), and then flatten them using the Accumulo column visibility class. + * Aggregate visibilities by concatenating them together with an AND ('&'), and then normalizing the resulting expression. * * @param visibilities * a set of valid visibility strings. - * @return a single visibility + * @return a combined {@link AccessExpression}, or {@code null} if there are no visibilities */ - Map generateCombinedVisibility(Set visibilities); + AccessExpression generateCombinedVisibility(Set visibilities); /** * Aggregate metadata from multiple sources, combining multi-valued entries into a flattened string diff --git a/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java b/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java index 79cb984e963..20e69b9342c 100644 --- a/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java +++ b/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java @@ -3,10 +3,12 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Map.Entry; +import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.data.ByteSequence; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.security.ColumnVisibility; @@ -18,7 +20,6 @@ import com.google.common.collect.Maps; import com.google.common.collect.Multimap; import com.google.common.collect.Multimaps; -import com.google.common.collect.Sets; import datawave.core.common.util.AccessExpressionUtil; import datawave.data.type.OneToManyNormalizerType; @@ -68,8 +69,8 @@ public class ValueToAttributes implements Function,Iterable markingFunctions, - boolean fromIndex) { + public ValueToAttributes(CompositeMetadata compositeMetadata, TypeMetadata typeMetadata, EventDataQueryFilter attrFilter, + MarkingFunctions markingFunctions, boolean fromIndex) { // in Java 25 we can create the attribute factory first this(new AttributeFactory(typeMetadata), compositeMetadata, attrFilter, markingFunctions, fromIndex); } @@ -226,7 +227,7 @@ public Attribute getFieldValue(String fieldName, Key k) { } public Attribute joinAttributes(String compositeName, Collection> in, boolean isOverloadedComposite, String separator) throws Exception { - Collection accessExpressions = Sets.newHashSet(); + Collection accessExpressions = new HashSet<>(); List dataList = new ArrayList<>(); long timestamp = 0; boolean toKeep = false; diff --git a/warehouse/query-core/src/main/java/datawave/query/table/parser/TermFrequencyKeyValueFactory.java b/warehouse/query-core/src/main/java/datawave/query/table/parser/TermFrequencyKeyValueFactory.java index 84a17312e9e..0eeac1c9597 100644 --- a/warehouse/query-core/src/main/java/datawave/query/table/parser/TermFrequencyKeyValueFactory.java +++ b/warehouse/query-core/src/main/java/datawave/query/table/parser/TermFrequencyKeyValueFactory.java @@ -16,7 +16,8 @@ public class TermFrequencyKeyValueFactory { - public static TermFrequencyKeyValue parse(Key key, Value value, Authorizations auths, MarkingFunctions markingFunctions) throws MarkingFunctions.Exception { + public static TermFrequencyKeyValue parse(Key key, Value value, Authorizations auths, MarkingFunctions markingFunctions) + throws MarkingFunctions.Exception { if (null == key) { throw new IllegalArgumentException("Cannot pass null key to TermFrequencyKeyValueFactory"); } diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java index 19898771eb6..230254408ab 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java @@ -14,6 +14,7 @@ import java.util.Map.Entry; import java.util.Set; import java.util.TreeSet; +import java.util.stream.Collectors; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.BatchScanner; @@ -477,12 +478,12 @@ public Map buildSummary(Iterator iter, long maxValues) { public static class Tuple { - private final MarkingFunctions tupleMarkingFunctions; + private final MarkingFunctions tupleMarkingFunctions; private long count = 0L; private long maxTimestamp = 0L; Set uniqueVisibilities = new HashSet<>(); - public Tuple(MarkingFunctions mf) { + public Tuple(MarkingFunctions mf) { tupleMarkingFunctions = mf; } @@ -508,8 +509,8 @@ public ColumnVisibility getColumnVisibility() { for (Text t : this.uniqueVisibilities) { columnVisibilities.add(new ColumnVisibility(t)); } - return AccessExpressionUtil.toColumnVisibility(tupleMarkingFunctions.combine( - columnVisibilities.stream().map(AccessExpressionUtil::toAccessExpression).collect(java.util.stream.Collectors.toSet()))); + return AccessExpressionUtil.toColumnVisibility(tupleMarkingFunctions + .combine(columnVisibilities.stream().map(AccessExpressionUtil::toAccessExpression).collect(Collectors.toList()))); } catch (MarkingFunctions.Exception e) { logger.error("Could not create combined column visibility for the count", e); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java index 0ef05e70c91..cbca9c07031 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java @@ -36,7 +36,7 @@ public class DocumentTransformer extends DocumentTransformerSupport,EventBase> implements WritesQueryMetrics, WritesResultCardinalities, Flushable { - private static final Logger log = Logger.getLogger(DocumentTransformerSupport.class); + private static final Logger log = Logger.getLogger(DocumentTransformer.class); /** * By default, assume each cell still has the visibility attached to it @@ -163,7 +163,7 @@ private EventBase _transform(Entry documentEntry) throws EmptyObje return output; } - protected EventBase buildResponse(Document document, Key documentKey, ColumnVisibility eventCV, String colf, String row, MarkingFunctions mf) + protected EventBase buildResponse(Document document, Key documentKey, ColumnVisibility eventCV, String colf, String row, MarkingFunctions mf) throws MarkingFunctions.Exception { Markings markings = mf.fromAccessExpression(AccessExpressionUtil.toAccessExpression(eventCV)); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformerSupport.java b/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformerSupport.java index 895bc6fe459..eb450ddd8b1 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformerSupport.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformerSupport.java @@ -2,6 +2,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -12,7 +13,6 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.builder.HashCodeBuilder; -import com.google.common.collect.Sets; import com.google.protobuf.InvalidProtocolBufferException; import datawave.core.query.cachedresults.CacheableLogic; @@ -34,7 +34,8 @@ public abstract class EdgeQueryTransformerSupport extends BaseQueryLogicTra protected ResponseObjectFactory responseObjectFactory; protected EdgeModelFields fields; - public EdgeQueryTransformerSupport(Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, EdgeModelFields fields) { + public EdgeQueryTransformerSupport(Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, + EdgeModelFields fields) { super(markingFunctions); this.responseObjectFactory = responseObjectFactory; auths = new Authorizations(settings.getQueryAuthorizations().split(",")); @@ -100,7 +101,7 @@ public BaseQueryResponse createResponse(List resultList) { try { EdgeQueryResponseBase response = responseObjectFactory.getEdgeQueryResponse(); - Set uniqueExpressions = Sets.newHashSet(); + Set uniqueExpressions = new HashSet<>(); for (Object result : resultList) { EdgeBase edge = (EdgeBase) result; Markings markings = edge.getMarkings(); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java index 9821ec80e6e..037bb1b99ef 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java @@ -149,7 +149,7 @@ protected Collection buildFacets(Key documentKey, String f return myFields; } - protected FacetsBase buildResponse(Document document, Key documentKey, ColumnVisibility eventCV, String colf, String row, MarkingFunctions mf) + protected FacetsBase buildResponse(Document document, Key documentKey, ColumnVisibility eventCV, String colf, String row, MarkingFunctions mf) throws MarkingFunctions.Exception { FacetsBase facetedResponse = responseObjectFactory.getFacets(); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/TagCloudTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/TagCloudTransformer.java index e40c85e8b26..351df1c5b3f 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/TagCloudTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/TagCloudTransformer.java @@ -2,11 +2,11 @@ import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.stream.Collectors; +import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.Authorizations; @@ -201,15 +201,12 @@ private void configureV2TagCloud(TagCloudBase base, TagCloud tagCloud) { } @SuppressWarnings("unchecked") - private void convertAndSetMarkings(TagCloudBase base, Map visibility) { - if (visibility != null && !visibility.isEmpty()) { - String expr = visibility.getOrDefault("visibility", ""); - if (!expr.isEmpty()) { - try { - base.setMarkings(markingFunctions.fromAccessExpression(MarkingFunctions.Default.ACCESS.newExpression(expr))); - } catch (MarkingFunctions.Exception e) { - log.warn("Could not convert tag cloud visibility to markings: {}", expr, e); - } + private void convertAndSetMarkings(TagCloudBase base, AccessExpression visibility) { + if (visibility != null && !visibility.getExpression().isEmpty()) { + try { + base.setMarkings(markingFunctions.fromAccessExpression(visibility)); + } catch (MarkingFunctions.Exception e) { + log.warn("Could not convert tag cloud visibility to markings: {}", visibility.getExpression(), e); } } } diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/RemoteEventQueryLogicIT.java b/warehouse/query-core/src/test/java/datawave/query/tables/RemoteEventQueryLogicIT.java index 93d046c9eb0..d35aac7bae8 100644 --- a/warehouse/query-core/src/test/java/datawave/query/tables/RemoteEventQueryLogicIT.java +++ b/warehouse/query-core/src/test/java/datawave/query/tables/RemoteEventQueryLogicIT.java @@ -161,7 +161,7 @@ public void testSocketTimeout() { assertTrue(r.getException() instanceof RemoteTimeoutQueryException); } - @Test(timeout = 30000) + @Test(timeout = 35000) public void testDefaultConnectionPoolTimeout() { RemoteHttpService remoteHttpService = (RemoteHttpService) logic.getRemoteQueryService(); RemoteHttpServiceConfiguration remoteConfig = remoteHttpService.getConfig(); diff --git a/web-services/metrics/src/main/java/datawave/query/metrics/ShardTableQueryMetricHandler.java b/web-services/metrics/src/main/java/datawave/query/metrics/ShardTableQueryMetricHandler.java index 304cac6efa4..d4b6ec5e4b1 100644 --- a/web-services/metrics/src/main/java/datawave/query/metrics/ShardTableQueryMetricHandler.java +++ b/web-services/metrics/src/main/java/datawave/query/metrics/ShardTableQueryMetricHandler.java @@ -24,8 +24,6 @@ import javax.enterprise.context.ApplicationScoped; import javax.inject.Inject; -import datawave.core.common.util.AccessExpressionUtil; -import datawave.marking.Markings; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; @@ -62,6 +60,7 @@ import datawave.core.common.connection.AccumuloConnectionFactory; import datawave.core.common.connection.AccumuloConnectionFactory.Priority; import datawave.core.common.logging.ThreadConfigurableLogger; +import datawave.core.common.util.AccessExpressionUtil; import datawave.core.query.cache.ResultsPage; import datawave.core.query.logic.QueryLogic; import datawave.core.query.logic.QueryLogicFactory; @@ -77,6 +76,7 @@ import datawave.ingest.mapreduce.job.BulkIngestKey; import datawave.ingest.mapreduce.job.writer.LiveContextWriter; import datawave.ingest.table.config.TableConfigHelper; +import datawave.marking.Markings; import datawave.microservice.query.Query; import datawave.microservice.query.QueryImpl; import datawave.microservice.query.QueryImpl.Parameter; From c315e824d208469ed2518df227423412314862fd Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Mon, 23 Mar 2026 16:11:36 -0400 Subject: [PATCH 05/63] added accumulo-access jar to classpath --- .../ingest-scripts/src/main/resources/bin/ingest/findJars.sh | 1 + .../ingest-scripts/src/main/resources/bin/ingest/ingest-libs.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/warehouse/ingest-scripts/src/main/resources/bin/ingest/findJars.sh b/warehouse/ingest-scripts/src/main/resources/bin/ingest/findJars.sh index 7a1a376e530..a7b487fc275 100644 --- a/warehouse/ingest-scripts/src/main/resources/bin/ingest/findJars.sh +++ b/warehouse/ingest-scripts/src/main/resources/bin/ingest/findJars.sh @@ -84,6 +84,7 @@ LUCENE_JAR=$LUCENE_JAR:$(findJar lucene-queryparser) LUCENE_JAR=$LUCENE_JAR:$(findJar lucene-analyzers-common) THRIFT_JAR=$(findJar libthrift) AC_CORE_JAR=$(findAccumuloJar accumulo-core) +AC_ACCESS_JAR=$(findJar accumulo-access) AC_SERVER_JAR=$(findAccumuloJar accumulo-server-base) AC_START_JAR=$(findAccumuloJar accumulo-start) AC_MAPRED_JAR=$(findAccumuloJar accumulo-hadoop-mapreduce) diff --git a/warehouse/ingest-scripts/src/main/resources/bin/ingest/ingest-libs.sh b/warehouse/ingest-scripts/src/main/resources/bin/ingest/ingest-libs.sh index b4fdb395d39..8eabe24b185 100755 --- a/warehouse/ingest-scripts/src/main/resources/bin/ingest/ingest-libs.sh +++ b/warehouse/ingest-scripts/src/main/resources/bin/ingest/ingest-libs.sh @@ -57,6 +57,7 @@ CLASSPATH=${CLASSPATH}:${LOG4J2_SLF4J_JAR} CLASSPATH=${CLASSPATH}:${LUCENE_JAR} CLASSPATH=${CLASSPATH}:${THRIFT_JAR} CLASSPATH=${CLASSPATH}:${AC_CORE_JAR} +CLASSPATH=${CLASSPATH}:${AC_ACCESS_JAR} CLASSPATH=${CLASSPATH}:${AC_MAPRED_JAR} CLASSPATH=${CLASSPATH}:${AC_SERVER_JAR} CLASSPATH=${CLASSPATH}:${AC_START_JAR} From acbc2434ee39b2058aefde95f441d44eea77cc67 Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Tue, 24 Mar 2026 10:36:43 -0400 Subject: [PATCH 06/63] temporarily pull snapshot deps from github packages --- core/in-memory-accumulo/pom.xml | 2 +- core/metrics-reporter/pom.xml | 2 +- core/utils/accumulo-utils/pom.xml | 2 +- core/utils/common-utils/pom.xml | 2 +- core/utils/metadata-utils/pom.xml | 2 +- microservices/configcheck/pom.xml | 2 +- microservices/microservice-parent/pom.xml | 2 +- microservices/microservice-service-parent/pom.xml | 2 +- microservices/services/accumulo/api/pom.xml | 2 +- microservices/services/accumulo/pom.xml | 2 +- microservices/services/accumulo/service/pom.xml | 2 +- microservices/services/audit/api/pom.xml | 2 +- microservices/services/audit/pom.xml | 2 +- microservices/services/audit/service/pom.xml | 2 +- microservices/services/authorization/api/pom.xml | 2 +- microservices/services/authorization/pom.xml | 2 +- microservices/services/authorization/service/pom.xml | 2 +- microservices/services/config/pom.xml | 2 +- microservices/services/dictionary/api/pom.xml | 2 +- microservices/services/dictionary/pom.xml | 2 +- microservices/services/dictionary/service/pom.xml | 2 +- microservices/services/file-provider/api/pom.xml | 2 +- microservices/services/file-provider/pom.xml | 2 +- microservices/services/file-provider/service/pom.xml | 2 +- microservices/services/hazelcast/client/pom.xml | 2 +- microservices/services/hazelcast/common/pom.xml | 2 +- microservices/services/hazelcast/pom.xml | 2 +- microservices/services/hazelcast/service/pom.xml | 2 +- microservices/services/mapreduce-query/api/pom.xml | 2 +- microservices/services/mapreduce-query/jobs/core/pom.xml | 2 +- microservices/services/mapreduce-query/layout-factory/pom.xml | 2 +- microservices/services/mapreduce-query/pom.xml | 2 +- microservices/services/mapreduce-query/service/pom.xml | 2 +- microservices/services/modification/api/pom.xml | 2 +- microservices/services/modification/pom.xml | 2 +- microservices/services/modification/service/pom.xml | 2 +- microservices/services/query-executor/api/pom.xml | 2 +- microservices/services/query-executor/pom.xml | 2 +- microservices/services/query-executor/service/pom.xml | 2 +- microservices/services/query-metric/api/pom.xml | 2 +- microservices/services/query-metric/pom.xml | 2 +- microservices/services/query-metric/service/pom.xml | 2 +- microservices/services/query/api/pom.xml | 2 +- microservices/services/query/pom.xml | 2 +- microservices/services/query/service/pom.xml | 2 +- microservices/starters/audit/pom.xml | 2 +- microservices/starters/cache/pom.xml | 2 +- microservices/starters/cached-results/pom.xml | 2 +- microservices/starters/datawave/pom.xml | 2 +- microservices/starters/metadata/pom.xml | 2 +- microservices/starters/query-metric/pom.xml | 2 +- microservices/starters/query/pom.xml | 2 +- pom.xml | 2 +- 53 files changed, 53 insertions(+), 53 deletions(-) diff --git a/core/in-memory-accumulo/pom.xml b/core/in-memory-accumulo/pom.xml index d4372bd8499..821df23e903 100644 --- a/core/in-memory-accumulo/pom.xml +++ b/core/in-memory-accumulo/pom.xml @@ -114,7 +114,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/core/metrics-reporter/pom.xml b/core/metrics-reporter/pom.xml index 475fe34bd5c..82bc6110ee0 100644 --- a/core/metrics-reporter/pom.xml +++ b/core/metrics-reporter/pom.xml @@ -105,7 +105,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/core/utils/accumulo-utils/pom.xml b/core/utils/accumulo-utils/pom.xml index fa2a2610d63..03d9a346ded 100644 --- a/core/utils/accumulo-utils/pom.xml +++ b/core/utils/accumulo-utils/pom.xml @@ -121,7 +121,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/core/utils/common-utils/pom.xml b/core/utils/common-utils/pom.xml index 97e5de2c0ea..4f411581fea 100644 --- a/core/utils/common-utils/pom.xml +++ b/core/utils/common-utils/pom.xml @@ -41,7 +41,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/core/utils/metadata-utils/pom.xml b/core/utils/metadata-utils/pom.xml index 944de13daf9..d089c11e3f6 100644 --- a/core/utils/metadata-utils/pom.xml +++ b/core/utils/metadata-utils/pom.xml @@ -104,7 +104,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/configcheck/pom.xml b/microservices/configcheck/pom.xml index 1980129e042..7ce10cf34ba 100644 --- a/microservices/configcheck/pom.xml +++ b/microservices/configcheck/pom.xml @@ -209,7 +209,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/microservice-parent/pom.xml b/microservices/microservice-parent/pom.xml index 55a8f5ee33c..82980cbc5ee 100644 --- a/microservices/microservice-parent/pom.xml +++ b/microservices/microservice-parent/pom.xml @@ -240,7 +240,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/microservice-service-parent/pom.xml b/microservices/microservice-service-parent/pom.xml index c3321d86f10..ac28b546adc 100644 --- a/microservices/microservice-service-parent/pom.xml +++ b/microservices/microservice-service-parent/pom.xml @@ -136,7 +136,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/accumulo/api/pom.xml b/microservices/services/accumulo/api/pom.xml index aa42f6e2c52..d7c77835a64 100644 --- a/microservices/services/accumulo/api/pom.xml +++ b/microservices/services/accumulo/api/pom.xml @@ -132,7 +132,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/accumulo/pom.xml b/microservices/services/accumulo/pom.xml index 77839795115..1b955053180 100644 --- a/microservices/services/accumulo/pom.xml +++ b/microservices/services/accumulo/pom.xml @@ -26,7 +26,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/accumulo/service/pom.xml b/microservices/services/accumulo/service/pom.xml index fca3f5ee1f8..836de2d9139 100644 --- a/microservices/services/accumulo/service/pom.xml +++ b/microservices/services/accumulo/service/pom.xml @@ -370,7 +370,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/audit/api/pom.xml b/microservices/services/audit/api/pom.xml index fa8058ce07a..e890fda23d4 100644 --- a/microservices/services/audit/api/pom.xml +++ b/microservices/services/audit/api/pom.xml @@ -71,7 +71,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/audit/pom.xml b/microservices/services/audit/pom.xml index 06f07c9c68b..ed3c94b5f55 100644 --- a/microservices/services/audit/pom.xml +++ b/microservices/services/audit/pom.xml @@ -26,7 +26,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/audit/service/pom.xml b/microservices/services/audit/service/pom.xml index cfdab2d5c54..529aba9bc89 100644 --- a/microservices/services/audit/service/pom.xml +++ b/microservices/services/audit/service/pom.xml @@ -441,7 +441,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/authorization/api/pom.xml b/microservices/services/authorization/api/pom.xml index a1769178b2a..a12c9bba99c 100644 --- a/microservices/services/authorization/api/pom.xml +++ b/microservices/services/authorization/api/pom.xml @@ -140,7 +140,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/authorization/pom.xml b/microservices/services/authorization/pom.xml index 2f5f20b5897..c4f990f101d 100644 --- a/microservices/services/authorization/pom.xml +++ b/microservices/services/authorization/pom.xml @@ -26,7 +26,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/authorization/service/pom.xml b/microservices/services/authorization/service/pom.xml index 361a9c6624b..695050e9a12 100644 --- a/microservices/services/authorization/service/pom.xml +++ b/microservices/services/authorization/service/pom.xml @@ -256,7 +256,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/config/pom.xml b/microservices/services/config/pom.xml index 04eb26e230b..432870d857f 100644 --- a/microservices/services/config/pom.xml +++ b/microservices/services/config/pom.xml @@ -138,7 +138,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/dictionary/api/pom.xml b/microservices/services/dictionary/api/pom.xml index 5fa0f3e29a2..60ab10091fb 100644 --- a/microservices/services/dictionary/api/pom.xml +++ b/microservices/services/dictionary/api/pom.xml @@ -106,7 +106,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/dictionary/pom.xml b/microservices/services/dictionary/pom.xml index cddd3cd4498..729973b3b78 100644 --- a/microservices/services/dictionary/pom.xml +++ b/microservices/services/dictionary/pom.xml @@ -26,7 +26,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/dictionary/service/pom.xml b/microservices/services/dictionary/service/pom.xml index 2cb915e104c..f70bf37f5eb 100644 --- a/microservices/services/dictionary/service/pom.xml +++ b/microservices/services/dictionary/service/pom.xml @@ -371,7 +371,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/file-provider/api/pom.xml b/microservices/services/file-provider/api/pom.xml index 056d4cc263c..e0ce6785c4c 100644 --- a/microservices/services/file-provider/api/pom.xml +++ b/microservices/services/file-provider/api/pom.xml @@ -22,7 +22,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/file-provider/pom.xml b/microservices/services/file-provider/pom.xml index 30a2ccf2c63..71c04ac6204 100644 --- a/microservices/services/file-provider/pom.xml +++ b/microservices/services/file-provider/pom.xml @@ -26,7 +26,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/file-provider/service/pom.xml b/microservices/services/file-provider/service/pom.xml index c639e2d9ce6..09e562ac03a 100644 --- a/microservices/services/file-provider/service/pom.xml +++ b/microservices/services/file-provider/service/pom.xml @@ -66,7 +66,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/hazelcast/client/pom.xml b/microservices/services/hazelcast/client/pom.xml index e2c5677ce6a..d758cf38332 100644 --- a/microservices/services/hazelcast/client/pom.xml +++ b/microservices/services/hazelcast/client/pom.xml @@ -78,7 +78,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/hazelcast/common/pom.xml b/microservices/services/hazelcast/common/pom.xml index 0d8905ba4a6..78e092dc680 100644 --- a/microservices/services/hazelcast/common/pom.xml +++ b/microservices/services/hazelcast/common/pom.xml @@ -102,7 +102,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/hazelcast/pom.xml b/microservices/services/hazelcast/pom.xml index 2f682a6d819..f470546bb2a 100644 --- a/microservices/services/hazelcast/pom.xml +++ b/microservices/services/hazelcast/pom.xml @@ -27,7 +27,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/hazelcast/service/pom.xml b/microservices/services/hazelcast/service/pom.xml index 15de2603381..536702fea22 100644 --- a/microservices/services/hazelcast/service/pom.xml +++ b/microservices/services/hazelcast/service/pom.xml @@ -98,7 +98,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/mapreduce-query/api/pom.xml b/microservices/services/mapreduce-query/api/pom.xml index 97f1c374ad7..5d722fd53a6 100644 --- a/microservices/services/mapreduce-query/api/pom.xml +++ b/microservices/services/mapreduce-query/api/pom.xml @@ -22,7 +22,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/mapreduce-query/jobs/core/pom.xml b/microservices/services/mapreduce-query/jobs/core/pom.xml index c4c38d98024..21fedd6065e 100644 --- a/microservices/services/mapreduce-query/jobs/core/pom.xml +++ b/microservices/services/mapreduce-query/jobs/core/pom.xml @@ -106,7 +106,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/mapreduce-query/layout-factory/pom.xml b/microservices/services/mapreduce-query/layout-factory/pom.xml index f1123c8fef9..de3dea9911c 100644 --- a/microservices/services/mapreduce-query/layout-factory/pom.xml +++ b/microservices/services/mapreduce-query/layout-factory/pom.xml @@ -31,7 +31,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/mapreduce-query/pom.xml b/microservices/services/mapreduce-query/pom.xml index 1acf64e05ba..2905b41643a 100644 --- a/microservices/services/mapreduce-query/pom.xml +++ b/microservices/services/mapreduce-query/pom.xml @@ -26,7 +26,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/mapreduce-query/service/pom.xml b/microservices/services/mapreduce-query/service/pom.xml index 672dbcb0ca4..d4c0578f458 100644 --- a/microservices/services/mapreduce-query/service/pom.xml +++ b/microservices/services/mapreduce-query/service/pom.xml @@ -265,7 +265,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/modification/api/pom.xml b/microservices/services/modification/api/pom.xml index ecfb10259ef..c243e3a75ef 100644 --- a/microservices/services/modification/api/pom.xml +++ b/microservices/services/modification/api/pom.xml @@ -76,7 +76,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/modification/pom.xml b/microservices/services/modification/pom.xml index 3af4b1f0dcf..a7613e4bb08 100644 --- a/microservices/services/modification/pom.xml +++ b/microservices/services/modification/pom.xml @@ -26,7 +26,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/modification/service/pom.xml b/microservices/services/modification/service/pom.xml index 14c64ee1816..a46db7530b4 100644 --- a/microservices/services/modification/service/pom.xml +++ b/microservices/services/modification/service/pom.xml @@ -250,7 +250,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/query-executor/api/pom.xml b/microservices/services/query-executor/api/pom.xml index 84313acbedf..3524463b8f5 100644 --- a/microservices/services/query-executor/api/pom.xml +++ b/microservices/services/query-executor/api/pom.xml @@ -86,7 +86,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/query-executor/pom.xml b/microservices/services/query-executor/pom.xml index 434b7be3d13..b2735ce95b6 100644 --- a/microservices/services/query-executor/pom.xml +++ b/microservices/services/query-executor/pom.xml @@ -26,7 +26,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/query-executor/service/pom.xml b/microservices/services/query-executor/service/pom.xml index bc7cd01bef2..d1cca7fd33d 100644 --- a/microservices/services/query-executor/service/pom.xml +++ b/microservices/services/query-executor/service/pom.xml @@ -554,7 +554,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/query-metric/api/pom.xml b/microservices/services/query-metric/api/pom.xml index c393a690205..11de312a191 100644 --- a/microservices/services/query-metric/api/pom.xml +++ b/microservices/services/query-metric/api/pom.xml @@ -229,7 +229,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/query-metric/pom.xml b/microservices/services/query-metric/pom.xml index aa1b6ab9bb3..7cc9d2d3154 100644 --- a/microservices/services/query-metric/pom.xml +++ b/microservices/services/query-metric/pom.xml @@ -26,7 +26,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/query-metric/service/pom.xml b/microservices/services/query-metric/service/pom.xml index 4623e1a5b89..56ffddb4bee 100644 --- a/microservices/services/query-metric/service/pom.xml +++ b/microservices/services/query-metric/service/pom.xml @@ -681,7 +681,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/query/api/pom.xml b/microservices/services/query/api/pom.xml index 21b88bf96b2..4e137af4e36 100644 --- a/microservices/services/query/api/pom.xml +++ b/microservices/services/query/api/pom.xml @@ -138,7 +138,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/query/pom.xml b/microservices/services/query/pom.xml index 7445827eff0..7adb505614d 100644 --- a/microservices/services/query/pom.xml +++ b/microservices/services/query/pom.xml @@ -26,7 +26,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/query/service/pom.xml b/microservices/services/query/service/pom.xml index 4ee665c5a92..c181e33b6c6 100644 --- a/microservices/services/query/service/pom.xml +++ b/microservices/services/query/service/pom.xml @@ -601,7 +601,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/starters/audit/pom.xml b/microservices/starters/audit/pom.xml index 866c815f03a..f1b05073a6a 100644 --- a/microservices/starters/audit/pom.xml +++ b/microservices/starters/audit/pom.xml @@ -173,7 +173,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/starters/cache/pom.xml b/microservices/starters/cache/pom.xml index 01ab2bcb467..42c4692d1a5 100644 --- a/microservices/starters/cache/pom.xml +++ b/microservices/starters/cache/pom.xml @@ -94,7 +94,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/starters/cached-results/pom.xml b/microservices/starters/cached-results/pom.xml index d160e2bff1c..f06f16b0190 100644 --- a/microservices/starters/cached-results/pom.xml +++ b/microservices/starters/cached-results/pom.xml @@ -300,7 +300,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/starters/datawave/pom.xml b/microservices/starters/datawave/pom.xml index 2dc04cc69e8..75080d06bdf 100644 --- a/microservices/starters/datawave/pom.xml +++ b/microservices/starters/datawave/pom.xml @@ -654,7 +654,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/starters/metadata/pom.xml b/microservices/starters/metadata/pom.xml index 0fe00a2e34a..09bd19c7a23 100644 --- a/microservices/starters/metadata/pom.xml +++ b/microservices/starters/metadata/pom.xml @@ -152,7 +152,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/starters/query-metric/pom.xml b/microservices/starters/query-metric/pom.xml index 551b6b71b12..71d5a06fd47 100644 --- a/microservices/starters/query-metric/pom.xml +++ b/microservices/starters/query-metric/pom.xml @@ -151,7 +151,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/starters/query/pom.xml b/microservices/starters/query/pom.xml index 628f46ffeee..4185cd7d7e5 100644 --- a/microservices/starters/query/pom.xml +++ b/microservices/starters/query/pom.xml @@ -520,7 +520,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/pom.xml b/pom.xml index ed1064f965d..62af22d512e 100644 --- a/pom.xml +++ b/pom.xml @@ -1601,7 +1601,7 @@ true - false + true github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave From d67a93340910036893133474ce03f8dc703c2ed7 Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Tue, 24 Mar 2026 14:34:56 -0400 Subject: [PATCH 07/63] remove accumulo-utils exclusions --- microservices/services/accumulo/api/pom.xml | 6 ------ microservices/services/query-metric/api/pom.xml | 6 ------ microservices/services/query-metric/service/pom.xml | 6 ------ microservices/starters/datawave/pom.xml | 6 ------ 4 files changed, 24 deletions(-) diff --git a/microservices/services/accumulo/api/pom.xml b/microservices/services/accumulo/api/pom.xml index d7c77835a64..1c823e594d2 100644 --- a/microservices/services/accumulo/api/pom.xml +++ b/microservices/services/accumulo/api/pom.xml @@ -51,12 +51,6 @@ gov.nsa.datawave.core accumulo-utils ${version.datawave} - - - com.google.guava - guava - - gov.nsa.datawave.core diff --git a/microservices/services/query-metric/api/pom.xml b/microservices/services/query-metric/api/pom.xml index 11de312a191..1d4b5acfd39 100644 --- a/microservices/services/query-metric/api/pom.xml +++ b/microservices/services/query-metric/api/pom.xml @@ -63,12 +63,6 @@ gov.nsa.datawave.core accumulo-utils ${version.datawave} - - - * - * - - gov.nsa.datawave.core diff --git a/microservices/services/query-metric/service/pom.xml b/microservices/services/query-metric/service/pom.xml index 56ffddb4bee..b5a35275860 100644 --- a/microservices/services/query-metric/service/pom.xml +++ b/microservices/services/query-metric/service/pom.xml @@ -135,12 +135,6 @@ gov.nsa.datawave.core accumulo-utils ${version.datawave} - - - * - * - - gov.nsa.datawave.core diff --git a/microservices/starters/datawave/pom.xml b/microservices/starters/datawave/pom.xml index 75080d06bdf..b57cd0e2f22 100644 --- a/microservices/starters/datawave/pom.xml +++ b/microservices/starters/datawave/pom.xml @@ -90,12 +90,6 @@ gov.nsa.datawave.core accumulo-utils ${version.datawave} - - - ch.qos.logback - * - - gov.nsa.datawave.core From 3f440bd658fe29675c66a34fedd6462e9031721d Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Tue, 24 Mar 2026 14:35:29 -0400 Subject: [PATCH 08/63] fix MetadataCachingTest --- .../microservice/querymetric/MetadataCachingTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/MetadataCachingTest.java b/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/MetadataCachingTest.java index cebdf520ada..2d346df6339 100644 --- a/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/MetadataCachingTest.java +++ b/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/MetadataCachingTest.java @@ -13,6 +13,7 @@ import org.apache.accumulo.core.security.Authorizations; import org.junit.Assert; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -79,6 +80,7 @@ public void VerifyMetadataMethodsCacheable() throws Exception { metadataHelper.getDatatypes(Collections.singleton("querymetrics")); metadataHelper.getFieldsForDatatype(LcNoDiacriticsType.class); metadataHelper.getFieldsToDatatypes(Collections.singleton("querymetrics")); + metadataHelper.getHiddenFields(Collections.singleton("querymetrics")); metadataHelper.getContentFields(Collections.singleton("querymetrics")); metadataHelper.getExpansionFields(Collections.singleton("querymetrics")); metadataHelper.getQueryModel(queryMetricHandlerProperties.getMetadataTableName(), "querymetrics"); @@ -110,7 +112,7 @@ public void VerifyMetadataMethodsCacheable() throws Exception { expectedCacheNames.addAll(getCacheNamesForCacheableMethods(AllFieldMetadataHelper.class)); expectedCacheNames.addAll(getCacheNamesForCacheableMethods(DateIndexHelper.class)); - Assert.assertEquals(expectedCacheNames, cacheNames); + Assertions.assertEquals(expectedCacheNames, cacheNames); } private List getCacheNamesForCacheableMethods(Class clazz) { From 5656425ad577f1576f310c79f0d6fc9fad82d569 Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Tue, 24 Mar 2026 17:34:53 -0400 Subject: [PATCH 09/63] fix markings serialization --- .../marking/AccessExpressionMarkings.java | 19 +++++++++++++++++++ .../main/java/datawave/marking/Markings.java | 3 +++ 2 files changed, 22 insertions(+) diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java index 0776ba8be6e..99438ffc453 100644 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java @@ -1,14 +1,23 @@ package datawave.marking; +import org.apache.accumulo.access.Access; import org.apache.accumulo.access.AccessExpression; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + import lombok.Builder; import lombok.Data; @Data @Builder +@JsonAutoDetect(getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE) public class AccessExpressionMarkings implements Markings { + private static final Access ACCESS = Access.builder().build(); + public static final String COLUMN_VISIBILITY_KEY = "columnVisibility"; + private final AccessExpression accessExpression; @Override @@ -20,4 +29,14 @@ public AccessExpression getMarkings() { public boolean isEmpty() { return accessExpression == null || accessExpression.getExpression().isEmpty(); } + + @JsonProperty(COLUMN_VISIBILITY_KEY) + public String getColumnVisibilityString() { + return accessExpression != null ? accessExpression.getExpression() : ""; + } + + @JsonCreator + public static AccessExpressionMarkings create(@JsonProperty(COLUMN_VISIBILITY_KEY) String cv) { + return AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression(cv != null ? cv : "")).build(); + } } diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/Markings.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/Markings.java index 113f9a52cca..3101a2e8354 100644 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/Markings.java +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/Markings.java @@ -1,5 +1,8 @@ package datawave.marking; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS) public interface Markings { T getMarkings(); From 09c390884cc2a6c310c146414e52471f262d7233 Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Wed, 25 Mar 2026 14:18:53 -0400 Subject: [PATCH 10/63] Markings refactoring --- core/cached-results/pom.xml | 5 ++ .../cachedresults/CacheableQueryRowImpl.java | 39 +++--------- .../CacheableQueryRowReader.java | 30 +++------ .../java/datawave/core/cache/ClassCache.java | 3 + .../common/util/AccessExpressionUtil.java | 61 ++++++++++++++++++ .../modification/MutableMetadataHandler.java | 10 +-- .../datawave/marking/MarkingFunctions.java | 63 ++++--------------- .../MarkingFunctionsDefaultCombineTest.java | 21 +++---- core/utils/metadata-utils/pom.xml | 5 ++ .../FrequencyMetadataAggregator.java | 11 +--- .../services/accumulo/service/pom.xml | 9 +++ .../accumulo/admin/AdminService.java | 3 +- .../accumulo/lookup/LookupService.java | 3 +- .../metadata/DefaultMetadataFieldScanner.java | 3 +- .../metadata/MetadataDescriptionsHelper.java | 6 +- .../querymetric/QueryMetricOperations.java | 2 +- .../handler/ShardTableQueryMetricHandler.java | 7 +-- .../data/config/ingest/CompositeIngest.java | 3 +- .../data/config/ingest/VirtualIngest.java | 3 +- .../handler/atom/AtomDataTypeHandler.java | 8 +-- .../atom/AtomErrorDataTypeHandler.java | 13 +--- .../dateindex/DateIndexDataTypeHandler.java | 3 +- .../edge/ProtobufEdgeDataTypeHandler.java | 16 ++--- .../handler/edge/define/EdgeDataBundle.java | 3 +- .../handler/error/ErrorDataTypeHandler.java | 15 +---- .../error/ErrorShardedDataTypeHandler.java | 11 +--- .../mapreduce/handler/facet/FacetHandler.java | 2 +- .../handler/shard/ShardedDataTypeHandler.java | 8 +-- .../ingest/csv/mr/input/CSVRecordReader.java | 3 +- warehouse/keyword-common/pom.xml | 5 ++ .../util/keyword/DefaultTagCloudUtils.java | 11 +++- ...eldIndexCountingIteratorPerVisibility.java | 6 +- .../GlobalIndexDateSummaryIterator.java | 7 ++- .../iterators/ResultCountingIterator.java | 6 +- .../query/attributes/AttributeBag.java | 3 +- .../datawave/query/attributes/Document.java | 6 +- .../query/common/grouping/GroupingUtils.java | 4 +- .../query/discovery/DiscoveryIterator.java | 7 ++- .../query/discovery/DiscoveryTransformer.java | 19 ++---- .../query/discovery/TermInfoAggregation.java | 13 ++-- .../iterators/FieldIndexCountingIterator.java | 6 +- .../query/jexl/DatawaveInterpreter.java | 5 +- .../query/predicate/ValueToAttributes.java | 4 +- .../table/parser/EventKeyValueFactory.java | 2 +- .../parser/TermFrequencyKeyValueFactory.java | 2 +- .../shard/CountAggregatingIterator.java | 4 +- .../shard/CountResultPostprocessor.java | 7 ++- .../shard/FieldIndexCountQueryLogic.java | 4 +- .../transformer/DocumentTransformer.java | 2 +- .../DocumentTransformerSupport.java | 2 +- .../transformer/EdgeQueryTransformer.java | 3 +- .../EdgeQueryTransformerSupport.java | 10 ++- .../transformer/EventQueryTransformer.java | 40 +++++------- .../query/transformer/FacetedTransformer.java | 20 +++--- .../FieldIndexCountQueryTransformer.java | 8 +-- .../GroupingDocumentTransformer.java | 2 +- .../ShardIndexQueryTransformer.java | 8 +-- .../ShardQueryCountTableTransformer.java | 8 +-- .../transformer/TagCloudTransformer.java | 7 +-- .../tables/content/ContentQueryLogicTest.java | 2 +- 60 files changed, 281 insertions(+), 321 deletions(-) diff --git a/core/cached-results/pom.xml b/core/cached-results/pom.xml index c9e7a44f4f3..6e53d7e5cc9 100644 --- a/core/cached-results/pom.xml +++ b/core/cached-results/pom.xml @@ -15,6 +15,11 @@ gov.nsa.datawave.core base-rest-responses + + gov.nsa.datawave.core + datawave-core-common-util + ${project.version} + gov.nsa.datawave.core type-utils diff --git a/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java b/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java index cba6612aafa..c11ef3da668 100644 --- a/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java +++ b/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java @@ -9,13 +9,13 @@ import java.util.Set; import java.util.TreeSet; -import org.apache.accumulo.access.AccessExpression; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.collect.Sets; +import datawave.core.common.util.AccessExpressionUtil; import datawave.data.type.Type; import datawave.marking.AccessExpressionMarkings; import datawave.marking.MarkingFunctions; @@ -60,22 +60,15 @@ public void addColumn(String columnName, TypedValue columnTypedValue, Markings datawaveType, String columnName, TypedVa if (!currMarkings.equals(markings)) { try { - List expressions = new ArrayList<>(); - expressions.add(markingFunctions.toAccessExpression(currMarkings)); - expressions.add(markingFunctions.toAccessExpression(markings)); - AccessExpression combined = markingFunctions.combine(expressions); - Markings combinedMarkings = markingFunctions.fromAccessExpression(combined); + Markings combinedMarkings = AccessExpressionUtil.combine(markingFunctions, List.of(currMarkings, markings)); // use combined marking as new markings columnMarkingsMap.put(columnName, combinedMarkings); @@ -285,12 +274,8 @@ public void setColFam(String colFam) { public void setMarkings(Markings markings) { // validate the markings - try { - markingFunctions.toAccessExpression(markings); - this.markings = markings; - } catch (MarkingFunctions.Exception e) { - LOGGER.error("Invalid markings {}", markings, e); - } + AccessExpressionUtil.toAccessExpression(markings); + this.markings = markings; } public void setColumnMarkingsMap(Map> columnMarkingsMap) { @@ -315,11 +300,7 @@ public String getColumnSecurityMarkingString(Map columnMap) { String v = columnColumnVisibilityMap.get(field); String mStr = ""; if (m != null && !m.isEmpty()) { - try { - mStr = markingFunctions.toAccessExpression(m).getExpression(); - } catch (MarkingFunctions.Exception e) { - LOGGER.error("could not serialize markings {}", m, e); - } + mStr = AccessExpressionUtil.toAccessExpression(m).getExpression(); } if (v == null) { combinedMap.put(field, mStr); diff --git a/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowReader.java b/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowReader.java index c65fd0bb004..c03388213f0 100644 --- a/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowReader.java +++ b/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowReader.java @@ -9,10 +9,10 @@ import java.util.Set; import java.util.TreeSet; -import org.apache.accumulo.access.AccessExpression; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import datawave.core.common.util.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.webservice.query.cachedresults.CacheableQueryRow; @@ -90,12 +90,7 @@ public static CacheableQueryRow createRow(ResultSet cachedRowSet, Set fi if (columnToIndexMap.get("_markings_") != null) { String mStr = cachedRowSet.getString(columnToIndexMap.get("_markings_")); if (mStr != null && !mStr.isEmpty()) { - try { - AccessExpression ae = MarkingFunctions.Default.ACCESS.newExpression(mStr); - cqfc.setMarkings(markingFunctions.fromAccessExpression(ae)); - } catch (MarkingFunctions.Exception e) { - log.error("could not deserialize markings {}", mStr, e); - } + cqfc.setMarkings(AccessExpressionUtil.fromAccessExpression(MarkingFunctions.Default.ACCESS.newExpression(mStr))); } } if (columnToIndexMap.get("_column_markings_") != null) { @@ -107,19 +102,14 @@ public static CacheableQueryRow createRow(ResultSet cachedRowSet, Set fi String columnName = entry.getKey(); String combinedString = entry.getValue(); int x = combinedString.lastIndexOf(':'); - try { - if (x >= 0) { - String markingStr = combinedString.substring(0, x); - AccessExpression ae = MarkingFunctions.Default.ACCESS.newExpression(markingStr); - columnMarkingsMap.put(columnName, markingFunctions.fromAccessExpression(ae)); - columnVisibilityMap.put(columnName, combinedString.substring(x + 1)); - } else { - AccessExpression ae = MarkingFunctions.Default.ACCESS.newExpression(combinedString); - columnMarkingsMap.put(columnName, markingFunctions.fromAccessExpression(ae)); - columnVisibilityMap.put(columnName, ""); - } - } catch (MarkingFunctions.Exception e) { - log.error("could not deserialize column markings for {}: {}", columnName, combinedString, e); + if (x >= 0) { + String markingStr = combinedString.substring(0, x); + columnMarkingsMap.put(columnName, AccessExpressionUtil.fromAccessExpression(MarkingFunctions.Default.ACCESS.newExpression(markingStr))); + columnVisibilityMap.put(columnName, combinedString.substring(x + 1)); + } else { + columnMarkingsMap.put(columnName, + AccessExpressionUtil.fromAccessExpression(MarkingFunctions.Default.ACCESS.newExpression(combinedString))); + columnVisibilityMap.put(columnName, ""); } } cqfc.setColumnMarkingsMap(columnMarkingsMap); diff --git a/core/common-util/src/main/java/datawave/core/cache/ClassCache.java b/core/common-util/src/main/java/datawave/core/cache/ClassCache.java index 6f6238a6c96..6c0b8665b6e 100644 --- a/core/common-util/src/main/java/datawave/core/cache/ClassCache.java +++ b/core/common-util/src/main/java/datawave/core/cache/ClassCache.java @@ -27,6 +27,9 @@ public interface ClassCache { * @param name * the class name * @return a {@link Class} + * + * @throws ClassNotFoundException + * if class not found */ Class get(String name) throws ClassNotFoundException; } diff --git a/core/common-util/src/main/java/datawave/core/common/util/AccessExpressionUtil.java b/core/common-util/src/main/java/datawave/core/common/util/AccessExpressionUtil.java index d0c9bab2c1f..298b4aca0f5 100644 --- a/core/common-util/src/main/java/datawave/core/common/util/AccessExpressionUtil.java +++ b/core/common-util/src/main/java/datawave/core/common/util/AccessExpressionUtil.java @@ -2,11 +2,16 @@ import static java.nio.charset.StandardCharsets.UTF_8; +import java.util.Collection; + import org.apache.accumulo.access.Access; import org.apache.accumulo.access.AccessExpression; +import org.apache.accumulo.access.ParsedAccessExpression; +import org.apache.accumulo.access.examples.ParseExamples; import org.apache.accumulo.core.security.ColumnVisibility; import datawave.marking.AccessExpressionMarkings; +import datawave.marking.MarkingFunctions; import datawave.marking.Markings; /** @@ -38,6 +43,28 @@ public static AccessExpression toAccessExpression(ColumnVisibility cv) { return ACCESS.newExpression(new String(expr, UTF_8)); } + /** + * Convert a {@link Markings} to an {@link AccessExpression}. + * + * @param markings + * assumed to be of type AccessExpressionMarkings + * @return the embedded access expression + */ + public static AccessExpression toAccessExpression(Markings markings) { + return (AccessExpression) markings.getMarkings(); + } + + /** + * Convert a {@link AccessExpression} to an {@link AccessExpressionMarkings}. + * + * @param expression + * The AccessExpression to embed in the wrapper class + * @return the wrapped access expression + */ + public static AccessExpressionMarkings fromAccessExpression(AccessExpression expression) { + return AccessExpressionMarkings.builder().accessExpression(expression).build(); + } + /** * Convert an {@link Markings} to a {@link ColumnVisibility}. * @@ -85,6 +112,23 @@ public static AccessExpression toAccessExpression(byte[] visibilityBytes) { return ACCESS.newExpression(new String(visibilityBytes, UTF_8)); } + /** + * Normalize an {@link AccessExpression} by deduplicating and sorting terms. This is the replacement for the old {@code ColumnVisibility.flatten()} + * behavior. + * + * @param expression + * the expression to normalize + * @return a normalized {@link AccessExpression} + */ + public static AccessExpression normalize(AccessExpression expression) { + String expr = expression.getExpression(); + if (expr.isEmpty()) { + return expression; + } + ParsedAccessExpression parsed = MarkingFunctions.Default.ACCESS.newParsedExpression(expr); + return MarkingFunctions.Default.ACCESS.newExpression(ParseExamples.normalize(parsed).expression); + } + /** * @return the shared {@link Access} instance used by this utility */ @@ -98,4 +142,21 @@ public static Access getAccess() { public static AccessExpression emptyExpression() { return EMPTY_EXPRESSION; } + + /** + * Helper to call {@link MarkingFunctions#combine(Collection)} with a wildcard-typed {@link MarkingFunctions}. This avoids the wildcard-capture issue that + * arises when callers hold a {@code MarkingFunctions} reference. + * + * @param markingFunctions + * the marking functions instance + * @param markings + * the markings to combine + * @return the combined markings + * @throws MarkingFunctions.Exception + * if the markings cannot be combined + */ + @SuppressWarnings({"unchecked", "rawtypes"}) + public static Markings combine(MarkingFunctions markingFunctions, Collection> markings) throws MarkingFunctions.Exception { + return ((MarkingFunctions) markingFunctions).combine(markings); + } } diff --git a/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java b/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java index 5e87c47f8c2..c4fd81680fd 100644 --- a/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java +++ b/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java @@ -323,7 +323,7 @@ private Markings convertFieldMarkings(Map rawMarkings) throws if (vis.isEmpty()) { return null; } - return markingFunctions.fromAccessExpression(MarkingFunctions.Default.ACCESS.newExpression(vis)); + return AccessExpressionUtil.fromAccessExpression(MarkingFunctions.Default.ACCESS.newExpression(vis)); } public void process(AccumuloClient client, ModificationRequestBase request, Map> mutableFieldList, Set userAuths, @@ -659,7 +659,7 @@ protected void insert(MultiTableBatchWriter writer, String shardId, String datat if (null == markings || markings.isEmpty()) throw new IllegalArgumentException("No security information specified. Security markings must be supplied"); - viz = AccessExpressionUtil.toColumnVisibility(markingFunctions.toAccessExpression(markings)); + viz = AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.toAccessExpression(markings)); } insert(writer, shardId, datatype, eventUid, viz, fieldName, fieldValue, ts, isIndexOnlyField, isIndexed, isReverseIndexed, dataTypes, false, @@ -863,14 +863,14 @@ protected List> getField(AccumuloClient client, Set markings = markingFunctions + Markings markings = AccessExpressionUtil .fromAccessExpression(AccessExpressionUtil.toAccessExpression(e.getKey().getColumnVisibilityParsed())); if (null != oldFieldMarkings && !oldFieldMarkings.equals(markings)) { log.trace("Skipping key that does not match with markings: {}", e.getKey()); diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java index cbe8a8d39b7..23a8fd538a8 100644 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java @@ -1,6 +1,5 @@ package datawave.marking; -import java.util.Arrays; import java.util.Collection; import java.util.HashSet; import java.util.Set; @@ -23,30 +22,7 @@ public interface MarkingFunctions> { - AccessExpression combine(Collection expressions) throws MarkingFunctions.Exception; - - T combine(T... markings) throws MarkingFunctions.Exception; - - AccessExpression toAccessExpression(Markings markings) throws MarkingFunctions.Exception; - - T fromAccessExpression(AccessExpression expression) throws MarkingFunctions.Exception; - - /** - * Normalize an {@link AccessExpression} by deduplicating and sorting terms. This is the replacement for the old {@code ColumnVisibility.flatten()} - * behavior. - * - * @param expression - * the expression to normalize - * @return a normalized {@link AccessExpression} - */ - static AccessExpression normalize(AccessExpression expression) { - String expr = expression.getExpression(); - if (expr.isEmpty()) { - return expression; - } - ParsedAccessExpression parsed = Default.ACCESS.newParsedExpression(expr); - return Default.ACCESS.newExpression(ParseExamples.normalize(parsed).expression); - } + T combine(Collection markings) throws MarkingFunctions.Exception; class Exception extends java.lang.Exception { @@ -73,23 +49,25 @@ public Exception(Throwable cause) { class Default implements MarkingFunctions { public static final Access ACCESS = Access.builder().build(); - public static final AccessExpression EMPTY_MARKINGS = ACCESS.newExpression(""); + public static final AccessExpressionMarkings EMPTY_MARKINGS = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression("")).build(); @Override - public AccessExpression combine(Collection expressions) { - // filter out any empty expressions and concatenate with '&' - // @formatter:off + public AccessExpressionMarkings combine(Collection markings) { - Set uniqueExpressions = new HashSet<>(expressions); + Set uniqueMarkings = new HashSet<>(markings); - if (uniqueExpressions.isEmpty()) { + if (uniqueMarkings.isEmpty()) { return EMPTY_MARKINGS; } - if (uniqueExpressions.size() == 1) { - return uniqueExpressions.stream().findFirst().get(); + if (uniqueMarkings.size() == 1) { + return uniqueMarkings.stream().findFirst().get(); } + Set uniqueExpressions = markings.stream().map(AccessExpressionMarkings::getMarkings).collect(Collectors.toSet()); + + // filter out any empty expressions and concatenate with '&' + // @formatter:off ParsedAccessExpression expression = ACCESS.newParsedExpression(uniqueExpressions .stream() .map(AccessExpression::getExpression) @@ -101,25 +79,8 @@ public AccessExpression combine(Collection expressions) { // normalize the Parsed Expression and then return a copy without the parse tree // FIXME: Using a beta of the accumulo-access API so temporarily using the normalize function from the example code - return ACCESS.newExpression(ParseExamples.normalize(expression).expression); - } - - @Override - public final AccessExpressionMarkings combine(AccessExpressionMarkings... markings) { - return AccessExpressionMarkings.builder().accessExpression(combine(Arrays.stream(markings).map(Markings::getMarkings).collect(Collectors.toSet()))) - .build(); + return AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression(ParseExamples.normalize(expression).expression)).build(); } - - @Override - public AccessExpression toAccessExpression(Markings markings) { - return (AccessExpression) markings.getMarkings(); - } - - @Override - public AccessExpressionMarkings fromAccessExpression(AccessExpression expression) { - return AccessExpressionMarkings.builder().accessExpression(expression).build(); - } - } /** diff --git a/core/utils/accumulo-utils/src/test/java/datawave/marking/MarkingFunctionsDefaultCombineTest.java b/core/utils/accumulo-utils/src/test/java/datawave/marking/MarkingFunctionsDefaultCombineTest.java index 6cf06e763fe..54f5230d4df 100644 --- a/core/utils/accumulo-utils/src/test/java/datawave/marking/MarkingFunctionsDefaultCombineTest.java +++ b/core/utils/accumulo-utils/src/test/java/datawave/marking/MarkingFunctionsDefaultCombineTest.java @@ -5,7 +5,6 @@ import java.util.List; import org.apache.accumulo.access.Access; -import org.apache.accumulo.access.AccessExpression; import org.junit.jupiter.api.Test; public class MarkingFunctionsDefaultCombineTest { @@ -15,22 +14,22 @@ public class MarkingFunctionsDefaultCombineTest { public void testCombineAnds() throws MarkingFunctions.Exception { MarkingFunctions markingFunctions = new MarkingFunctions.Default(); - AccessExpression oneAnna = ACCESS.newExpression("A&B"); - AccessExpression twoAnna = ACCESS.newExpression("A&C"); + AccessExpressionMarkings oneAnna = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression("A&B")).build(); + AccessExpressionMarkings twoAnna = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression("A&C")).build(); - AccessExpression combined = markingFunctions.combine(List.of(oneAnna, twoAnna)); - assertEquals(ACCESS.newExpression("A&B&C"), combined); + AccessExpressionMarkings combined = markingFunctions.combine(List.of(oneAnna, twoAnna)); + assertEquals(AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression("A&B&C")).build(), combined); } @Test public void testCombineOrs() throws MarkingFunctions.Exception { MarkingFunctions markingFunctions = new MarkingFunctions.Default(); - AccessExpression oneOr = ACCESS.newExpression("A|B"); - AccessExpression twoOr = ACCESS.newExpression("A|C"); + AccessExpressionMarkings oneOr = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression("A|B")).build(); + AccessExpressionMarkings twoOr = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression("A|C")).build(); - AccessExpression combined = markingFunctions.combine(List.of(oneOr, twoOr)); - assertEquals(ACCESS.newExpression("(A|B)&(A|C)"), combined); + AccessExpressionMarkings combined = markingFunctions.combine(List.of(oneOr, twoOr)); + assertEquals(AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression("(A|B)&(A|C)")).build(), combined); } @Test @@ -41,7 +40,7 @@ public void testCombineMapsOfAnds() throws MarkingFunctions.Exception { AccessExpressionMarkings cv2 = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression("A&C")).build(); AccessExpressionMarkings expected = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression("A&B&C")).build(); - assertEquals(expected, markingFunctions.combine(cv1, cv2)); + assertEquals(expected, markingFunctions.combine(List.of(cv1, cv2))); } @Test @@ -52,6 +51,6 @@ public void testCombineMapsOfOrs() throws MarkingFunctions.Exception { AccessExpressionMarkings cv2 = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression("A|C")).build(); AccessExpressionMarkings expected = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression("(A|B)&(A|C)")).build(); - assertEquals(expected, markingFunctions.combine(cv1, cv2)); + assertEquals(expected, markingFunctions.combine(List.of(cv1, cv2))); } } diff --git a/core/utils/metadata-utils/pom.xml b/core/utils/metadata-utils/pom.xml index d089c11e3f6..279ca3488df 100644 --- a/core/utils/metadata-utils/pom.xml +++ b/core/utils/metadata-utils/pom.xml @@ -59,6 +59,11 @@ gov.nsa.datawave.core common-utils + + gov.nsa.datawave.core + datawave-core-common-util + ${project.version} + gov.nsa.datawave.core type-utils diff --git a/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java b/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java index 94b74e7ef22..156535fc1f2 100644 --- a/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java +++ b/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java @@ -1,7 +1,5 @@ package datawave.iterators; -import static java.nio.charset.StandardCharsets.UTF_8; - import java.io.IOException; import java.util.Arrays; import java.util.Collection; @@ -14,8 +12,6 @@ import java.util.TreeMap; import java.util.stream.Collectors; -import org.apache.accumulo.access.Access; -import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.data.ByteSequence; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; @@ -34,6 +30,7 @@ import com.google.common.base.Splitter; +import datawave.core.common.util.AccessExpressionUtil; import datawave.iterator.ReducingIterator; import datawave.marking.MarkingFunctions; import datawave.query.model.DateFrequencyMap; @@ -589,10 +586,8 @@ private Map buildCacheEntries() { private ColumnVisibility combineAllVisibilities() { Set visibilities = visibilityToDateFrequencies.keySet(); try { - Access access = Access.builder().build(); - AccessExpression combined = markingFunctions - .combine(visibilities.stream().map(cv -> access.newExpression(new String(cv.getExpression(), UTF_8))).collect(Collectors.toSet())); - return new ColumnVisibility(combined.getExpression()); + return AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(markingFunctions, visibilities.stream() + .map(cv -> AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(cv))).collect(Collectors.toSet()))); } catch (MarkingFunctions.Exception e) { log.error("Failed to combine visibilities " + visibilities); throw new IllegalArgumentException("Failed to combine visibilities " + visibilities, e); diff --git a/microservices/services/accumulo/service/pom.xml b/microservices/services/accumulo/service/pom.xml index 836de2d9139..ee4fb5fb79a 100644 --- a/microservices/services/accumulo/service/pom.xml +++ b/microservices/services/accumulo/service/pom.xml @@ -69,6 +69,11 @@ base-rest-responses ${version.datawave} + + gov.nsa.datawave.core + datawave-core-common-util + ${version.datawave} + gov.nsa.datawave.core type-utils @@ -229,6 +234,10 @@ gov.nsa.datawave.core base-rest-responses + + gov.nsa.datawave.core + datawave-core-common-util + gov.nsa.datawave.core type-utils diff --git a/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/admin/AdminService.java b/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/admin/AdminService.java index 717f6965bd2..75537241973 100644 --- a/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/admin/AdminService.java +++ b/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/admin/AdminService.java @@ -41,6 +41,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; +import datawave.core.common.util.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.webservice.query.util.OptionallyEncodedString; @@ -570,7 +571,7 @@ public ValidateVisibilityResponse validateVisibilities(String[] visibilityArray) vis.setValid(false); visibilityList.add(vis); try { - Markings markings = markingFunctions.fromAccessExpression(MarkingFunctions.Default.ACCESS.newExpression(v)); + Markings markings = AccessExpressionUtil.fromAccessExpression(MarkingFunctions.Default.ACCESS.newExpression(v)); vis.setVisibility(v); vis.setValid(true); vis.setMarkings(markings); diff --git a/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/lookup/LookupService.java b/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/lookup/LookupService.java index ee538a75b72..82f263f2bbc 100644 --- a/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/lookup/LookupService.java +++ b/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/lookup/LookupService.java @@ -34,6 +34,7 @@ import com.google.common.base.Preconditions; import datawave.accumulo.util.security.UserAuthFunctions; +import datawave.core.common.util.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.marking.SecurityMarking; @@ -218,7 +219,7 @@ public LookupResponse lookup(LookupRequest request, DatawaveUserDetails currentU continue; } - final Markings markings = markingFunctions + final Markings markings = AccessExpressionUtil .fromAccessExpression(MarkingFunctions.Default.ACCESS.newExpression(k.getColumnVisibility().toString())); final KeyBase responseKey = responseObjectFactory.createKey(); diff --git a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java index 68dc223f7ef..cce616eb3b2 100644 --- a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java +++ b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java @@ -22,6 +22,7 @@ import com.google.common.collect.Maps; +import datawave.core.common.util.AccessExpressionUtil; import datawave.data.ColumnFamilyConstants; import datawave.marking.MarkingFunctions; import datawave.microservice.Connection; @@ -276,7 +277,7 @@ private void setDescriptions() throws MarkingFunctions.Exception { DefaultDescription description = responseObjectFactory.getDescription(); description.setDescription(currValue.toString()); description.setMarkings( - markingFunctions.fromAccessExpression(MarkingFunctions.Default.ACCESS.newExpression(currKey.getColumnVisibility().toString()))); + AccessExpressionUtil.fromAccessExpression(MarkingFunctions.Default.ACCESS.newExpression(currKey.getColumnVisibility().toString()))); currField.getDescriptions().add(description); } diff --git a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java index 0fc059d783d..dcc40bd904b 100644 --- a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java +++ b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java @@ -128,7 +128,7 @@ public void setDescriptions(MetadataEntry entry, Set bw = accumuloClient.createBatchWriter(metadataTableName, bwConfig); Mutation m = new Mutation(entry.getFieldName()); for (DescriptionBase desc : descs) { - ColumnVisibility cv = AccessExpressionUtil.toColumnVisibility(markingFunctions.toAccessExpression(desc.getMarkings())); + ColumnVisibility cv = AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.toAccessExpression(desc.getMarkings())); m.put(ColumnFamilyConstants.COLF_DESC, new Text(entry.getDatatype()), cv, new Value(desc.getDescription().getBytes(UTF_8))); } bw.addMutation(m); @@ -162,7 +162,7 @@ public void removeDescription(MetadataEntry entry, DescriptionBase desc) BatchWriterConfig bwConfig = new BatchWriterConfig().setMaxMemory(10000L).setMaxLatency(100L, TimeUnit.MILLISECONDS).setMaxWriteThreads(1); bw = accumuloClient.createBatchWriter(metadataTableName, bwConfig); Mutation m = new Mutation(entry.getFieldName()); - ColumnVisibility cv = AccessExpressionUtil.toColumnVisibility(this.markingFunctions.toAccessExpression(desc.getMarkings())); + ColumnVisibility cv = AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.toAccessExpression(desc.getMarkings())); m.putDelete(ColumnFamilyConstants.COLF_DESC, new Text(entry.getDatatype()), cv); bw.addMutation(m); } finally { @@ -203,6 +203,6 @@ protected SetMultimap loadDescriptions() throws TableNotFoun } private Markings getMarkings(Key k) throws MarkingFunctions.Exception { - return markingFunctions.fromAccessExpression(AccessExpressionUtil.toAccessExpression(k.getColumnVisibilityParsed())); + return AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(k.getColumnVisibilityParsed())); } } diff --git a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java index d0c82dcd3b5..28a04c760ea 100644 --- a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java +++ b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java @@ -479,7 +479,7 @@ private boolean isMetricsAdministrator(DatawaveUserDetails currentUser) { private boolean canViewMetric(DatawaveUserDetails currentUser, BaseQueryMetric metric) throws Exception { boolean userCanViewMetric = true; - ColumnVisibility columnVisibility = AccessExpressionUtil.toColumnVisibility(markingFunctions.toAccessExpression(metric.getMarkings())); + ColumnVisibility columnVisibility = AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.toAccessExpression(metric.getMarkings())); for (DatawaveUser user : currentUser.getProxiedUsers()) { Authorizations authorizations = new Authorizations(user.getAuths().toArray(new String[0])); VisibilityEvaluator visibilityEvaluator = new VisibilityEvaluator(authorizations); diff --git a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java index 7144b764567..8ee1f0d176f 100644 --- a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java +++ b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java @@ -370,12 +370,7 @@ protected Multimap getEntries(ContentIndexingColumnBasedHan // get markings from metric, otherwise use the default markings Markings markings = updatedQueryMetric.getMarkings(); if (markings != null && !markings.isEmpty()) { - try { - event.setVisibility(AccessExpressionUtil.toColumnVisibility(markingFunctions.toAccessExpression(markings))); - } catch (MarkingFunctions.Exception e) { - log.error(e.getMessage(), e); - event.setVisibility(this.queryMetricHandlerProperties.getDefaultMetricVisibility()); - } + event.setVisibility(AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.toAccessExpression(markings))); } else { event.setVisibility(this.queryMetricHandlerProperties.getDefaultMetricVisibility()); } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CompositeIngest.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CompositeIngest.java index 62056a598b8..f19a848bac8 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CompositeIngest.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CompositeIngest.java @@ -25,6 +25,7 @@ import com.google.common.collect.Multimap; import com.google.common.collect.Sets; +import datawave.core.common.util.AccessExpressionUtil; import datawave.ingest.data.Type; import datawave.ingest.data.config.GroupedNormalizedContentInterface; import datawave.ingest.data.config.NormalizedContentInterface; @@ -305,7 +306,7 @@ protected Markings mergeMarkings(Markings markings1, Markings markings2 if (markings1 instanceof AccessExpressionMarkings && markings2 instanceof AccessExpressionMarkings) { AccessExpressionMarkings aem1 = (AccessExpressionMarkings) markings1; AccessExpressionMarkings aem2 = (AccessExpressionMarkings) markings2; - return markingFunctions.fromAccessExpression(markingFunctions.combine(List.of(aem1.getMarkings(), aem2.getMarkings()))); + return AccessExpressionUtil.combine(markingFunctions, List.of(aem1, aem2)); } else { throw new RuntimeException( String.format("Unknown markings class %s or %s", markings1.getClass().getName(), markings1.getClass().getName())); diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java index eee3e6ae3bd..436d0c07464 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java @@ -22,6 +22,7 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Multimap; +import datawave.core.common.util.AccessExpressionUtil; import datawave.data.type.NoOpType; import datawave.ingest.data.Type; import datawave.ingest.data.config.GroupedNormalizedContentInterface; @@ -358,7 +359,7 @@ protected Markings mergeMarkings(Markings markings1, Markings markings2 if (markings1 instanceof AccessExpressionMarkings && markings2 instanceof AccessExpressionMarkings) { AccessExpressionMarkings aem1 = (AccessExpressionMarkings) markings1; AccessExpressionMarkings aem2 = (AccessExpressionMarkings) markings2; - return markingFunctions.fromAccessExpression(markingFunctions.combine(List.of(aem1.getMarkings(), aem2.getMarkings()))); + return AccessExpressionUtil.combine(markingFunctions, List.of(aem1, aem2)); } else { throw new RuntimeException( String.format("Unknown markings class %s or %s", markings1.getClass().getName(), markings1.getClass().getName())); diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomDataTypeHandler.java index 486ec19201d..40e52b5b545 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomDataTypeHandler.java @@ -261,11 +261,7 @@ public Key createKey(String fieldName, String fieldValue, String columnQualifier protected String getColumnQualifier(RawRecordContainer event, NormalizedContentInterface value) { Markings markings = value.getMarkings(); if (markings != null && !markings.isEmpty()) { - try { - return markingFunctions.toAccessExpression(markings).getExpression(); - } catch (MarkingFunctions.Exception e) { - throw new RuntimeException("Cannot convert record-level markings into a column visibility", e); - } + return AccessExpressionUtil.toAccessExpression(markings).getExpression(); } return flatten(event.getVisibility()); } @@ -278,7 +274,7 @@ protected String getColumnQualifier(RawRecordContainer event, NormalizedContentI * @return the normalized visibility string */ protected String flatten(ColumnVisibility vis) { - return MarkingFunctions.normalize(AccessExpressionUtil.toAccessExpression(vis)).getExpression(); + return AccessExpressionUtil.normalize(AccessExpressionUtil.toAccessExpression(vis)).getExpression(); } } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomErrorDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomErrorDataTypeHandler.java index 47c04df0ba1..b6c22b5d336 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomErrorDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomErrorDataTypeHandler.java @@ -17,6 +17,7 @@ import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; +import datawave.core.common.util.AccessExpressionUtil; import datawave.data.hash.UID; import datawave.ingest.config.IngestConfigurationFactory; import datawave.ingest.data.RawDataErrorNames; @@ -77,11 +78,7 @@ public long process(KEYIN key, RawRecordContainer record, Multimap markings = value.getMarkings(); if (markings != null && !markings.isEmpty()) { - try { - return markingFunctions.toAccessExpression(markings).getExpression(); - } catch (MarkingFunctions.Exception e) { - throw new RuntimeException("Cannot convert record-level markings into a column visibility", e); - } + return AccessExpressionUtil.toAccessExpression(markings).getExpression(); } ColumnVisibility vis = event.getVisibility(); if (vis != null) { diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/dateindex/DateIndexDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/dateindex/DateIndexDataTypeHandler.java index 65c507e7067..232b1726cd3 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/dateindex/DateIndexDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/dateindex/DateIndexDataTypeHandler.java @@ -40,7 +40,6 @@ import datawave.ingest.mapreduce.job.BulkIngestKey; import datawave.ingest.metadata.RawRecordMetadata; import datawave.ingest.table.aggregator.DateIndexDateAggregator; -import datawave.marking.MarkingFunctions; /** *

@@ -367,7 +366,7 @@ private Value createDateIndexValue(int shard) { * @return the flattened visibility */ protected byte[] flatten(ColumnVisibility vis) { - return MarkingFunctions.normalize(AccessExpressionUtil.toAccessExpression(vis)).getExpression().getBytes(UTF_8); + return AccessExpressionUtil.normalize(AccessExpressionUtil.toAccessExpression(vis)).getExpression().getBytes(UTF_8); } public Text getDateIndexTableName() { diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDataTypeHandler.java index fb8fa5e1a73..b0e609942ab 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDataTypeHandler.java @@ -1334,15 +1334,11 @@ public static Map> getLocalityGroups(Configuration conf) { */ @SuppressWarnings("unchecked") protected Text getVisibility(Markings markings, RawRecordContainer event) { - try { - if (null == markings || markings.isEmpty()) { - return new Text(flatten(event.getVisibility())); - } else { - AccessExpression ae = MarkingFunctions.normalize(markingFunctions.toAccessExpression(markings)); - return new Text(ae.getExpression().getBytes(UTF_8)); - } - } catch (MarkingFunctions.Exception e) { - throw new RuntimeException("Cannot convert markings into column visibility", e); + if (null == markings || markings.isEmpty()) { + return new Text(flatten(event.getVisibility())); + } else { + AccessExpression ae = AccessExpressionUtil.normalize(AccessExpressionUtil.toAccessExpression(markings)); + return new Text(ae.getExpression().getBytes(UTF_8)); } } @@ -1370,7 +1366,7 @@ protected Text getDurationVisibility(EdgeDataBundle value) { * @return the flattened visibility */ protected byte[] flatten(ColumnVisibility vis) { - return MarkingFunctions.normalize(AccessExpressionUtil.toAccessExpression(vis)).getExpression().getBytes(UTF_8); + return AccessExpressionUtil.normalize(AccessExpressionUtil.toAccessExpression(vis)).getExpression().getBytes(UTF_8); } @Override diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/EdgeDataBundle.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/EdgeDataBundle.java index 413569db5a7..b00c4bd48b2 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/EdgeDataBundle.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/EdgeDataBundle.java @@ -9,6 +9,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import datawave.core.common.util.AccessExpressionUtil; import datawave.edge.util.EdgeKey; import datawave.edge.util.EdgeValue.EdgeValueBuilder; import datawave.edge.util.EdgeValueHelper; @@ -157,7 +158,7 @@ protected void initMarkings(Markings markings1, Markings markings2) { if (markings1 instanceof AccessExpressionMarkings && markings2 instanceof AccessExpressionMarkings) { AccessExpressionMarkings aem1 = (AccessExpressionMarkings) markings1; AccessExpressionMarkings aem2 = (AccessExpressionMarkings) markings2; - this.markings = markingFunctions.fromAccessExpression(markingFunctions.combine(List.of(aem1.getMarkings(), aem2.getMarkings()))); + this.markings = AccessExpressionUtil.combine(markingFunctions, List.of(aem1, aem2)); } else { throw new RuntimeException( String.format("Unknown markings class %s or %s", markings1.getClass().getName(), markings1.getClass().getName())); diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorDataTypeHandler.java index 52dfc0f6fcc..33a033df521 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorDataTypeHandler.java @@ -135,11 +135,7 @@ public void setup(TaskAttemptContext context) { this.conf = context.getConfiguration(); - try { - defaultVisibility = markingFunctions.toAccessExpression(markingsHelper.getDefaultMarkings()).getExpression().getBytes(UTF_8); - } catch (MarkingFunctions.Exception e) { - throw new IllegalArgumentException("Failed to convert default markings to an AccessExpression.", e); - } + defaultVisibility = AccessExpressionUtil.toAccessExpression(markingsHelper.getDefaultMarkings()).getExpression().getBytes(UTF_8); // Initialize a UID builder based on the configuration uidBuilder = UID.builder(conf); @@ -322,12 +318,7 @@ private Key createKey(String row, Text colf, Text colq, byte[] vis, long ts) { protected byte[] getVisibility(RawRecordContainer event, NormalizedContentInterface value) { byte[] visibility; if (value != null && value.getMarkings() != null && !value.getMarkings().isEmpty()) { - try { - visibility = markingFunctions.toAccessExpression(value.getMarkings()).getExpression().getBytes(UTF_8); - } catch (MarkingFunctions.Exception e) { - log.error("Failed to create visibility from markings, using default", e); - visibility = defaultVisibility; - } + visibility = AccessExpressionUtil.toAccessExpression(value.getMarkings()).getExpression().getBytes(UTF_8); } else if (event.getVisibility() != null) { visibility = flatten(event.getVisibility()); } else { @@ -344,7 +335,7 @@ protected byte[] getVisibility(RawRecordContainer event, NormalizedContentInterf * @return the normalized visibility bytes */ protected byte[] flatten(ColumnVisibility vis) { - return MarkingFunctions.normalize(AccessExpressionUtil.toAccessExpression(vis)).getExpression().getBytes(UTF_8); + return AccessExpressionUtil.normalize(AccessExpressionUtil.toAccessExpression(vis)).getExpression().getBytes(UTF_8); } @Override diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java index 984d80fadd4..a8210eac01e 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java @@ -24,6 +24,7 @@ import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; +import datawave.core.common.util.AccessExpressionUtil; import datawave.data.hash.UID; import datawave.ingest.config.IngestConfiguration; import datawave.ingest.config.IngestConfigurationFactory; @@ -43,7 +44,6 @@ import datawave.ingest.mapreduce.handler.shard.ShardedDataTypeHandler; import datawave.ingest.mapreduce.job.BulkIngestKey; import datawave.ingest.mapreduce.job.writer.ContextWriter; -import datawave.marking.MarkingFunctions; /** * Handler that take events with processing errors or fatal errors and dumps them into a processing error table. This table will be used for subsequent @@ -158,7 +158,7 @@ public void setup(TaskAttemptContext context) { setShardDictionaryIndexTableName(tableName == null ? null : new Text(tableName)); try { - defaultVisibility = markingFunctions.toAccessExpression(markingsHelper.getDefaultMarkings()).getExpression().getBytes(UTF_8); + defaultVisibility = AccessExpressionUtil.toAccessExpression(markingsHelper.getDefaultMarkings()).getExpression().getBytes(UTF_8); } catch (Exception e) { throw new IllegalArgumentException("Failed to parse security marking configuration", e); } @@ -349,12 +349,7 @@ public static void getStackTrace(DataOutputBuffer buffer, Throwable e) { public byte[] getVisibility(RawRecordContainer event, NormalizedContentInterface value) { byte[] visibility; if (value != null && value.getMarkings() != null && !value.getMarkings().isEmpty()) { - try { - visibility = markingFunctions.toAccessExpression(value.getMarkings()).getExpression().getBytes(UTF_8); - } catch (MarkingFunctions.Exception e) { - log.error("Failed to create visibility from markings, using default", e); - visibility = defaultVisibility; - } + visibility = AccessExpressionUtil.toAccessExpression(value.getMarkings()).getExpression().getBytes(UTF_8); } else if (event.getVisibility() != null) { visibility = flatten(event.getVisibility()); } else { diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/facet/FacetHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/facet/FacetHandler.java index dc86577dd46..74cd2c8db39 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/facet/FacetHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/facet/FacetHandler.java @@ -195,7 +195,7 @@ public RawRecordMetadata getMetadata() { } protected byte[] flatten(ColumnVisibility vis) { - return MarkingFunctions.normalize(AccessExpressionUtil.toAccessExpression(vis)).getExpression().getBytes(UTF_8); + return AccessExpressionUtil.normalize(AccessExpressionUtil.toAccessExpression(vis)).getExpression().getBytes(UTF_8); } @Override diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java index d57146ac6de..1cecd7329f4 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java @@ -1135,11 +1135,7 @@ public int getOffsetForYearIndex(String shard) { public byte[] getVisibility(RawRecordContainer event, NormalizedContentInterface value) { Markings markings = value.getMarkings(); if (markings != null && !markings.isEmpty()) { - try { - return markingFunctions.toAccessExpression(markings).getExpression().getBytes(UTF_8); - } catch (MarkingFunctions.Exception e) { - throw new RuntimeException("Cannot convert record-level markings into a column visibility", e); - } + return AccessExpressionUtil.toAccessExpression(markings).getExpression().getBytes(UTF_8); } return flatten(event.getVisibility()); } @@ -1152,7 +1148,7 @@ public byte[] getVisibility(RawRecordContainer event, NormalizedContentInterface * @return the normalized visibility bytes */ protected byte[] flatten(ColumnVisibility vis) { - return MarkingFunctions.normalize(AccessExpressionUtil.toAccessExpression(vis)).getExpression().getBytes(UTF_8); + return AccessExpressionUtil.normalize(AccessExpressionUtil.toAccessExpression(vis)).getExpression().getBytes(UTF_8); } /** diff --git a/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java b/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java index 47beef190e7..14708dc40c1 100644 --- a/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java +++ b/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java @@ -16,6 +16,7 @@ import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; +import datawave.core.common.util.AccessExpressionUtil; import datawave.data.hash.UID; import datawave.ingest.config.IngestConfiguration; import datawave.ingest.config.IngestConfigurationFactory; @@ -126,7 +127,7 @@ public RawRecordContainer getEvent() { protected void decorateEvent() { if (securityMarkingExpression != null) { try { - AccessExpression ae = MarkingFunctions.normalize(MarkingFunctions.Default.ACCESS.newExpression(securityMarkingExpression)); + AccessExpression ae = AccessExpressionUtil.normalize(MarkingFunctions.Default.ACCESS.newExpression(securityMarkingExpression)); event.setSecurityMarkings(AccessExpressionMarkings.builder().accessExpression(ae).build()); } catch (Exception e) { log.error("Could not set security markings for the event", e); diff --git a/warehouse/keyword-common/pom.xml b/warehouse/keyword-common/pom.xml index 28e13e0f316..f3a2b18f4b4 100644 --- a/warehouse/keyword-common/pom.xml +++ b/warehouse/keyword-common/pom.xml @@ -21,5 +21,10 @@ datawave-core ${project.version} + + gov.nsa.datawave.core + datawave-core-common-util + ${project.version} + diff --git a/warehouse/keyword-common/src/main/java/datawave/util/keyword/DefaultTagCloudUtils.java b/warehouse/keyword-common/src/main/java/datawave/util/keyword/DefaultTagCloudUtils.java index f6983c178e5..b0ff4c0e086 100644 --- a/warehouse/keyword-common/src/main/java/datawave/util/keyword/DefaultTagCloudUtils.java +++ b/warehouse/keyword-common/src/main/java/datawave/util/keyword/DefaultTagCloudUtils.java @@ -12,7 +12,9 @@ import org.apache.accumulo.access.AccessExpression; import org.apache.commons.lang3.StringUtils; +import datawave.core.common.util.AccessExpressionUtil; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; /** * Default implementations for pluggable utilities for generating tag clouds, includes mechanisms to partition keywords into separate tag clouds, combine or @@ -27,8 +29,13 @@ public AccessExpression generateCombinedVisibility(Set visibilities) { if (visibilities == null || visibilities.isEmpty()) { return null; } - Set expressions = visibilities.stream().map(MarkingFunctions.Default.ACCESS::newExpression).collect(Collectors.toSet()); - return new MarkingFunctions.Default().combine(expressions); + Set> markings = visibilities.stream().map(v -> AccessExpressionUtil.fromAccessExpression(MarkingFunctions.Default.ACCESS.newExpression(v))) + .collect(Collectors.toSet()); + try { + return AccessExpressionUtil.toAccessExpression(AccessExpressionUtil.combine(new MarkingFunctions.Default(), markings)); + } catch (MarkingFunctions.Exception e) { + throw new IllegalArgumentException("Failed to combine visibilities", e); + } } @Override diff --git a/warehouse/query-core/src/main/java/datawave/core/iterators/FieldIndexCountingIteratorPerVisibility.java b/warehouse/query-core/src/main/java/datawave/core/iterators/FieldIndexCountingIteratorPerVisibility.java index b96a188f1ae..d0414d7b1ea 100644 --- a/warehouse/query-core/src/main/java/datawave/core/iterators/FieldIndexCountingIteratorPerVisibility.java +++ b/warehouse/query-core/src/main/java/datawave/core/iterators/FieldIndexCountingIteratorPerVisibility.java @@ -12,6 +12,7 @@ import java.util.TimeZone; import java.util.TreeMap; import java.util.TreeSet; +import java.util.stream.Collectors; import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.data.ArrayByteSequence; @@ -566,8 +567,9 @@ private Map buildReturnKeys() { ColumnVisibility cv = null; try { // Calculate the columnVisibility for this key from the combiner. - cv = AccessExpressionUtil.toColumnVisibility(MarkingFunctions.Factory.createMarkingFunctions().combine( - columnVisibilities.stream().map(AccessExpressionUtil::toAccessExpression).collect(java.util.stream.Collectors.toSet()))); + cv = AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(MarkingFunctions.Factory.createMarkingFunctions(), + columnVisibilities.stream().map(c -> AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(c))) + .collect(Collectors.toSet()))); } catch (Exception e) { log.error("Could not create combined columnVisibility for the count", e); return null; diff --git a/warehouse/query-core/src/main/java/datawave/core/iterators/GlobalIndexDateSummaryIterator.java b/warehouse/query-core/src/main/java/datawave/core/iterators/GlobalIndexDateSummaryIterator.java index c1852d12d8a..3c72d342d6e 100644 --- a/warehouse/query-core/src/main/java/datawave/core/iterators/GlobalIndexDateSummaryIterator.java +++ b/warehouse/query-core/src/main/java/datawave/core/iterators/GlobalIndexDateSummaryIterator.java @@ -8,6 +8,7 @@ import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; +import java.util.stream.Collectors; import org.apache.accumulo.core.data.ByteSequence; import org.apache.accumulo.core.data.Key; @@ -257,8 +258,10 @@ public Map getKeyValues() throws IOException { Set columnVisibilities = this.columnVisibilitiesMap.get(datatype); // Note that the access controls found in the combined ColumnVisibility will be pulled out appropriately here - ColumnVisibility cv = AccessExpressionUtil.toColumnVisibility(MarkingFunctions.Factory.createMarkingFunctions().combine( - columnVisibilities.stream().map(AccessExpressionUtil::toAccessExpression).collect(java.util.stream.Collectors.toSet()))); + ColumnVisibility cv = AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine( + MarkingFunctions.Factory.createMarkingFunctions(), + columnVisibilities.stream().map(c -> AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(c))) + .collect(Collectors.toSet()))); // Create a new Key compatible with the shardIndex key format Key k = new Key(this.fieldValue, this.fieldName, this.date + '\0' + datatype, new String(cv.getExpression())); diff --git a/warehouse/query-core/src/main/java/datawave/core/iterators/ResultCountingIterator.java b/warehouse/query-core/src/main/java/datawave/core/iterators/ResultCountingIterator.java index bd537b7b90f..81e6e5fd237 100644 --- a/warehouse/query-core/src/main/java/datawave/core/iterators/ResultCountingIterator.java +++ b/warehouse/query-core/src/main/java/datawave/core/iterators/ResultCountingIterator.java @@ -7,6 +7,7 @@ import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import org.apache.accumulo.core.client.BatchScanner; import org.apache.accumulo.core.data.ByteSequence; @@ -210,8 +211,9 @@ public Value getTopValue() { ColumnVisibility cv = null; try { - cv = AccessExpressionUtil.toColumnVisibility(MarkingFunctions.Factory.createMarkingFunctions() - .combine(columnVisibilities.stream().map(AccessExpressionUtil::toAccessExpression).collect(java.util.stream.Collectors.toSet()))); + cv = AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(MarkingFunctions.Factory.createMarkingFunctions(), + columnVisibilities.stream().map(c -> AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(c))) + .collect(Collectors.toSet()))); } catch (MarkingFunctions.Exception e) { log.error("Could not create combined columnVisibility for the count", e); return null; diff --git a/warehouse/query-core/src/main/java/datawave/query/attributes/AttributeBag.java b/warehouse/query-core/src/main/java/datawave/query/attributes/AttributeBag.java index f65437ebe7c..baf05bf2811 100644 --- a/warehouse/query-core/src/main/java/datawave/query/attributes/AttributeBag.java +++ b/warehouse/query-core/src/main/java/datawave/query/attributes/AttributeBag.java @@ -81,7 +81,8 @@ private void updateMetadata() { protected ColumnVisibility combineAndSetColumnVisibilities(Collection>> attributes) throws Exception { Collection accessExpressions = attributes.stream().map(attr -> AccessExpressionUtil.toAccessExpression(attr.getColumnVisibility())) .collect(Collectors.toSet()); - return AccessExpressionUtil.toColumnVisibility(MarkingFunctions.Factory.createMarkingFunctions().combine(accessExpressions)); + return AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(MarkingFunctions.Factory.createMarkingFunctions(), + accessExpressions.stream().map(AccessExpressionUtil::fromAccessExpression).collect(Collectors.toList()))); } private long updateTimestamps() { diff --git a/warehouse/query-core/src/main/java/datawave/query/attributes/Document.java b/warehouse/query-core/src/main/java/datawave/query/attributes/Document.java index e2696ab998a..5269e3c6330 100644 --- a/warehouse/query-core/src/main/java/datawave/query/attributes/Document.java +++ b/warehouse/query-core/src/main/java/datawave/query/attributes/Document.java @@ -87,11 +87,7 @@ public MarkingFunctions getMarkingFunctions() { } public Markings getMarkings() { - try { - MarkingFunctions markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); - return markingFunctions.fromAccessExpression(AccessExpressionUtil.toAccessExpression(getColumnVisibility())); - } catch (MarkingFunctions.Exception e) {} - return null; + return AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(getColumnVisibility())); } public Document() { diff --git a/warehouse/query-core/src/main/java/datawave/query/common/grouping/GroupingUtils.java b/warehouse/query-core/src/main/java/datawave/query/common/grouping/GroupingUtils.java index 647c8451144..0f49c41aabc 100644 --- a/warehouse/query-core/src/main/java/datawave/query/common/grouping/GroupingUtils.java +++ b/warehouse/query-core/src/main/java/datawave/query/common/grouping/GroupingUtils.java @@ -7,6 +7,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.data.Key; @@ -46,7 +47,8 @@ public enum AverageAggregatorWriteFormat { */ public static ColumnVisibility combineVisibilities(Collection expressions, MarkingFunctions markingFunctions, boolean failOnError) { try { - return AccessExpressionUtil.toColumnVisibility(markingFunctions.combine(expressions)); + return AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(markingFunctions, + expressions.stream().map(AccessExpressionUtil::fromAccessExpression).collect(Collectors.toList()))); } catch (MarkingFunctions.Exception e) { if (failOnError) { throw new IllegalArgumentException("Unable to combine access expressions: " + expressions, e); diff --git a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java index 89e8054f884..bfcca1f9840 100644 --- a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java +++ b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java @@ -163,11 +163,12 @@ private DiscoveredThing aggregate(Collection termEntries) { } else { // Otherwise, combine the visibilities, and return the aggregated result. try { - ColumnVisibility visibility = AccessExpressionUtil.toColumnVisibility(markingFunctions - .combine(visibilities.stream().map(AccessExpressionUtil::toAccessExpression).collect(java.util.stream.Collectors.toSet()))); + ColumnVisibility visibility = AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(markingFunctions, + visibilities.stream().map(cv -> AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(cv))) + .collect(Collectors.toSet()))); MapWritable countsByVis = new MapWritable(); visibilityToCounts.forEach((key, value) -> countsByVis.put(new Text(key), new LongWritable(value))); - String normalizedVis = MarkingFunctions.normalize(AccessExpressionUtil.toAccessExpression(visibility)).getExpression(); + String normalizedVis = AccessExpressionUtil.normalize(AccessExpressionUtil.toAccessExpression(visibility)).getExpression(); return new DiscoveredThing(term, first.getField(), first.getDatatype(), date, normalizedVis, count, countsByVis); } catch (Exception e) { if (log.isTraceEnabled()) { diff --git a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryTransformer.java index 18c5d67b4bc..e37fbebe5ba 100644 --- a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryTransformer.java @@ -14,7 +14,6 @@ import datawave.core.query.logic.BaseQueryLogic; import datawave.core.query.logic.BaseQueryLogicTransformer; import datawave.marking.MarkingFunctions; -import datawave.marking.MarkingFunctions.Exception; import datawave.marking.Markings; import datawave.microservice.query.Query; import datawave.query.model.QueryModel; @@ -47,12 +46,8 @@ public EventBase transform(DiscoveredThing thing) { Preconditions.checkNotNull(thing, "Received a null object to transform!"); EventBase event = this.responseObjectFactory.getEvent(); - Markings markings; - try { - markings = this.markingFunctions.fromAccessExpression(AccessExpressionUtil.toAccessExpression(new ColumnVisibility(thing.getColumnVisibility()))); - } catch (Exception e) { - throw new RuntimeException("could not parse to markings: " + thing.getColumnVisibility()); - } + Markings markings = AccessExpressionUtil + .fromAccessExpression(AccessExpressionUtil.toAccessExpression(new ColumnVisibility(thing.getColumnVisibility()))); event.setMarkings(markings); List fields = new ArrayList<>(); @@ -68,13 +63,9 @@ public EventBase transform(DiscoveredThing thing) { // If requested return counts separated by colvis, all counts by colvis could be > total record count if (thing.getCountsByColumnVisibility() != null && !thing.getCountsByColumnVisibility().isEmpty()) { for (Map.Entry entry : thing.getCountsByColumnVisibility().entrySet()) { - try { - Markings eMarkings = this.markingFunctions - .fromAccessExpression(AccessExpressionUtil.toAccessExpression(new ColumnVisibility(entry.getKey().toString()))); - fields.add(this.makeField("RECORD COUNT", eMarkings, entry.getKey().toString(), 0L, entry.getValue().toString())); - } catch (Exception e) { - throw new RuntimeException("could not parse to markings: " + thing.getColumnVisibility()); - } + Markings eMarkings = AccessExpressionUtil + .fromAccessExpression(AccessExpressionUtil.toAccessExpression(new ColumnVisibility(entry.getKey().toString()))); + fields.add(this.makeField("RECORD COUNT", eMarkings, entry.getKey().toString(), 0L, entry.getValue().toString())); } } else { diff --git a/warehouse/query-core/src/main/java/datawave/query/discovery/TermInfoAggregation.java b/warehouse/query-core/src/main/java/datawave/query/discovery/TermInfoAggregation.java index a0292fcb03c..90cd4759527 100644 --- a/warehouse/query-core/src/main/java/datawave/query/discovery/TermInfoAggregation.java +++ b/warehouse/query-core/src/main/java/datawave/query/discovery/TermInfoAggregation.java @@ -5,6 +5,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import java.util.stream.Collectors; import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.hadoop.io.MapWritable; @@ -74,8 +75,8 @@ public DiscoveredThing apply(Collection from) { chosenCount = showReferenceCountInsteadOfTermCount ? referenceCount : termCount; try { - MarkingFunctions.Factory.createMarkingFunctions().fromAccessExpression(AccessExpressionUtil.toAccessExpression(ti.vis)); // just to test - // parsing + AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(ti.vis)); // just to test + // parsing columnVisibilities.add(ti.vis); // Keep track of counts for individual vis @@ -109,8 +110,10 @@ public DiscoveredThing apply(Collection from) { ColumnVisibility columnVisibility = null; try { - columnVisibility = AccessExpressionUtil.toColumnVisibility(MarkingFunctions.Factory.createMarkingFunctions().combine( - columnVisibilities.stream().map(AccessExpressionUtil::toAccessExpression).collect(java.util.stream.Collectors.toSet()))); + columnVisibility = AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(MarkingFunctions.Factory.createMarkingFunctions(), + columnVisibilities.stream() + .map(cv -> AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(cv))) + .collect(Collectors.toSet()))); } catch (Exception e) { log.warn("Invalid columnvisibility after combining!", e); @@ -123,7 +126,7 @@ public DiscoveredThing apply(Collection from) { } return new DiscoveredThing(term, field, type, date, - MarkingFunctions.normalize(AccessExpressionUtil.toAccessExpression(columnVisibility)).getExpression(), count, countsByVis); + AccessExpressionUtil.normalize(AccessExpressionUtil.toAccessExpression(columnVisibility)).getExpression(), count, countsByVis); } } } diff --git a/warehouse/query-core/src/main/java/datawave/query/iterators/FieldIndexCountingIterator.java b/warehouse/query-core/src/main/java/datawave/query/iterators/FieldIndexCountingIterator.java index 5e326f3df1f..5c364e6aac5 100644 --- a/warehouse/query-core/src/main/java/datawave/query/iterators/FieldIndexCountingIterator.java +++ b/warehouse/query-core/src/main/java/datawave/query/iterators/FieldIndexCountingIterator.java @@ -11,6 +11,7 @@ import java.util.Set; import java.util.TimeZone; import java.util.TreeSet; +import java.util.stream.Collectors; import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.data.ArrayByteSequence; @@ -588,8 +589,9 @@ private Key buildReturnKey() { } ColumnVisibility cv; try { - cv = AccessExpressionUtil.toColumnVisibility(MarkingFunctions.Factory.createMarkingFunctions() - .combine(columnVisibilities.stream().map(AccessExpressionUtil::toAccessExpression).collect(java.util.stream.Collectors.toSet()))); + cv = AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(MarkingFunctions.Factory.createMarkingFunctions(), + columnVisibilities.stream().map(c -> AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(c))) + .collect(Collectors.toSet()))); } catch (MarkingFunctions.Exception e) { log.error("Could not combine visibilities: " + visibilitySet + " " + e); return null; diff --git a/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java b/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java index 95bf56ab54f..979e63569fb 100644 --- a/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java +++ b/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java @@ -15,6 +15,7 @@ import java.util.Map; import java.util.Set; import java.util.SortedSet; +import java.util.stream.Collectors; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.security.ColumnVisibility; @@ -470,8 +471,8 @@ private void addHitsForFunction(String field, ASTFunctionNode node, HitListArith } try { - cv = AccessExpressionUtil.toColumnVisibility(MarkingFunctionsFactory.createMarkingFunctions() - .combine(cvs.stream().map(AccessExpressionUtil::toAccessExpression).collect(java.util.stream.Collectors.toSet()))); + cv = AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(MarkingFunctionsFactory.createMarkingFunctions(), cvs.stream() + .map(c -> AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(c))).collect(Collectors.toSet()))); } catch (MarkingFunctions.Exception e) { log.error("Failed to combine column visibilities while generating HIT_TERM for phrase function for field [" + field + "]"); log.error("msg: ", e); diff --git a/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java b/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java index 20e69b9342c..2e1384f92a6 100644 --- a/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java +++ b/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java @@ -7,6 +7,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.stream.Collectors; import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.data.ByteSequence; @@ -292,7 +293,8 @@ public Attribute joinAttributes(String compositeName, Collection if (log.isDebugEnabled()) { log.debug("dataList is " + dataList); } - ColumnVisibility combinedColumnVisibility = AccessExpressionUtil.toColumnVisibility(this.markingFunctions.combine(accessExpressions)); + ColumnVisibility combinedColumnVisibility = AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(this.markingFunctions, + accessExpressions.stream().map(AccessExpressionUtil::fromAccessExpression).collect(Collectors.toList()))); metadata = new Key(metadata.getRow(), metadata.getColumnFamily(), new Text(), combinedColumnVisibility, timestamp); if (dataList.size() == 1) { return this.attrFactory.create(compositeName, dataList.get(0), metadata, toKeep, true); diff --git a/warehouse/query-core/src/main/java/datawave/query/table/parser/EventKeyValueFactory.java b/warehouse/query-core/src/main/java/datawave/query/table/parser/EventKeyValueFactory.java index 731d3698f6c..b9f18991897 100644 --- a/warehouse/query-core/src/main/java/datawave/query/table/parser/EventKeyValueFactory.java +++ b/warehouse/query-core/src/main/java/datawave/query/table/parser/EventKeyValueFactory.java @@ -41,7 +41,7 @@ public static EventKeyValue parse(Key key, Value value, Authorizations auths, Ma protected static void parseColumnVisibility(EventKeyValue event, Key key, Authorizations auths, MarkingFunctions markingFunctions) throws MarkingFunctions.Exception { - event.setMarkings(markingFunctions.fromAccessExpression(AccessExpressionUtil.toAccessExpression(new ColumnVisibility(key.getColumnVisibility())))); + event.setMarkings(AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(new ColumnVisibility(key.getColumnVisibility())))); } public static class EventKeyValue { diff --git a/warehouse/query-core/src/main/java/datawave/query/table/parser/TermFrequencyKeyValueFactory.java b/warehouse/query-core/src/main/java/datawave/query/table/parser/TermFrequencyKeyValueFactory.java index 0eeac1c9597..5a77edfed4a 100644 --- a/warehouse/query-core/src/main/java/datawave/query/table/parser/TermFrequencyKeyValueFactory.java +++ b/warehouse/query-core/src/main/java/datawave/query/table/parser/TermFrequencyKeyValueFactory.java @@ -58,7 +58,7 @@ public static TermFrequencyKeyValue parse(Key key, Value value, Authorizations a protected static void parseColumnVisibility(TermFrequencyKeyValue tfkv, Key key, Authorizations auths, MarkingFunctions markingFunctions) throws MarkingFunctions.Exception { - tfkv.setMarkings(markingFunctions.fromAccessExpression(AccessExpressionUtil.toAccessExpression(key.getColumnVisibilityParsed()))); + tfkv.setMarkings(AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(key.getColumnVisibilityParsed()))); } public static class TermFrequencyKeyValue extends EventKeyValue { diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountAggregatingIterator.java b/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountAggregatingIterator.java index 66cbf00a1dd..42f2753e8f6 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountAggregatingIterator.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountAggregatingIterator.java @@ -154,8 +154,8 @@ public Object getAggregatedEvent() { private ColumnVisibility getCombinedColumnVisibility() { try { - return AccessExpressionUtil.toColumnVisibility(markingFunctions - .combine(columnVisibilities.stream().map(AccessExpressionUtil::toAccessExpression).collect(Collectors.toSet()))); + return AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(markingFunctions, columnVisibilities.stream() + .map(c -> AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(c))).collect(Collectors.toSet()))); } catch (Exception e) { log.error("Could not combine columnVisibilities for the count", e); return null; diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountResultPostprocessor.java b/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountResultPostprocessor.java index 4ad6952936f..af96643963e 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountResultPostprocessor.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountResultPostprocessor.java @@ -5,6 +5,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.log4j.Logger; @@ -71,8 +72,10 @@ public void apply(List results) { if (success) { ColumnVisibility columnVisibility = null; try { - columnVisibility = AccessExpressionUtil.toColumnVisibility(markingFunctions.combine( - columnVisibilities.stream().map(AccessExpressionUtil::toAccessExpression).collect(java.util.stream.Collectors.toSet()))); + columnVisibility = AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(markingFunctions, + columnVisibilities.stream() + .map(cv -> AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(cv))) + .collect(Collectors.toSet()))); } catch (Exception e) { log.error("Could not create combined columnVisibilities for the count", e); } diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java index 230254408ab..50d65e29053 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java @@ -509,8 +509,8 @@ public ColumnVisibility getColumnVisibility() { for (Text t : this.uniqueVisibilities) { columnVisibilities.add(new ColumnVisibility(t)); } - return AccessExpressionUtil.toColumnVisibility(tupleMarkingFunctions - .combine(columnVisibilities.stream().map(AccessExpressionUtil::toAccessExpression).collect(Collectors.toList()))); + return AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(tupleMarkingFunctions, columnVisibilities.stream() + .map(c -> AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(c))).collect(Collectors.toList()))); } catch (MarkingFunctions.Exception e) { logger.error("Could not create combined column visibility for the count", e); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java index cbca9c07031..78a5563a7bb 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java @@ -166,7 +166,7 @@ private EventBase _transform(Entry documentEntry) throws EmptyObje protected EventBase buildResponse(Document document, Key documentKey, ColumnVisibility eventCV, String colf, String row, MarkingFunctions mf) throws MarkingFunctions.Exception { - Markings markings = mf.fromAccessExpression(AccessExpressionUtil.toAccessExpression(eventCV)); + Markings markings = AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(eventCV)); EventBase event = null; final Collection> documentFields = buildDocumentFields(documentKey, null, document, eventCV, mf); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformerSupport.java b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformerSupport.java index b3306bcb556..6482e264f1b 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformerSupport.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformerSupport.java @@ -466,7 +466,7 @@ protected Collection> buildDocumentFields(Key documentKey, String f // Use the markings on the Field if we're returning the markings to the client if (!this.reducedResponse) { try { - Markings markings = markingFunctions.fromAccessExpression(AccessExpressionUtil.toAccessExpression(attr.getColumnVisibility())); + Markings markings = AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(attr.getColumnVisibility())); FieldBase field = this.makeField(fieldName, markings, attr.getColumnVisibility(), attr.getTimestamp(), attr.getData()); myFields.add(field); } catch (Exception ex) { diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformer.java index 55addc5c354..f399ae12f0e 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformer.java @@ -41,7 +41,8 @@ public EdgeBase transform(Entry entry) { boolean statsEdge = edgeKey.isStatsKey(); try { - Markings markings = markingFunctions.fromAccessExpression(AccessExpressionUtil.toAccessExpression(new ColumnVisibility(edgeKey.getColvis()))); + Markings markings = AccessExpressionUtil + .fromAccessExpression(AccessExpressionUtil.toAccessExpression(new ColumnVisibility(edgeKey.getColvis()))); edge.setMarkings(markings); edge.setEdgeType(edgeKey.getType()); edge.setEdgeRelationship(edgeKey.getRelationship()); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformerSupport.java b/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformerSupport.java index eb450ddd8b1..40cfaef17bf 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformerSupport.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformerSupport.java @@ -7,7 +7,6 @@ import java.util.Map; import java.util.Set; -import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.Authorizations; import org.apache.commons.lang.StringUtils; @@ -15,6 +14,7 @@ import com.google.protobuf.InvalidProtocolBufferException; +import datawave.core.common.util.AccessExpressionUtil; import datawave.core.query.cachedresults.CacheableLogic; import datawave.core.query.logic.BaseQueryLogicTransformer; import datawave.edge.model.EdgeModelFields; @@ -101,16 +101,14 @@ public BaseQueryResponse createResponse(List resultList) { try { EdgeQueryResponseBase response = responseObjectFactory.getEdgeQueryResponse(); - Set uniqueExpressions = new HashSet<>(); + Set> uniqueMarkings = new HashSet<>(); for (Object result : resultList) { EdgeBase edge = (EdgeBase) result; - Markings markings = edge.getMarkings(); - uniqueExpressions.add(this.markingFunctions.toAccessExpression(markings)); + uniqueMarkings.add(edge.getMarkings()); response.addEdge(edge); } - AccessExpression combinedExpression = this.markingFunctions.combine(uniqueExpressions); - response.setMarkings(this.markingFunctions.fromAccessExpression(combinedExpression)); + response.setMarkings(AccessExpressionUtil.combine(this.markingFunctions, uniqueMarkings)); return response; } catch (Exception ex) { throw new RuntimeException("could not handle markings in resultList ", ex); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java index f197e1273db..c57721ea546 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java @@ -8,7 +8,6 @@ import java.util.Map.Entry; import java.util.Set; -import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.ColumnVisibility; @@ -54,13 +53,8 @@ public EventBase transform(Entry entry) { } EventBase event = this.responseObjectFactory.getEvent(); - Markings markings = null; - try { - markings = this.markingFunctions.fromAccessExpression(AccessExpressionUtil.toAccessExpression(new ColumnVisibility(key.getColumnVisibility()))); - } catch (Exception e) { - log.error("could not translate " + key.getColumnVisibility() + " to markings, skipping entry"); - return null; - } + Markings markings = AccessExpressionUtil + .fromAccessExpression(AccessExpressionUtil.toAccessExpression(new ColumnVisibility(key.getColumnVisibility()))); if (null == markings || markings.isEmpty()) { // can't process this one because we did not have valid security markings log.error("Transformer visibility interpreter was null, skipping entry"); @@ -88,9 +82,9 @@ public EventBase transform(Entry entry) { String origFieldName = null; String fieldName = null; - // Hold unique AccessExpressions and merge them at the end + // Hold unique Markings and merge them at the end // for the overall event visibility. - Set expressionsToMerge = new HashSet<>(); + Set> markingsToMerge = new HashSet<>(); for (Entry> e : eventFields.asMap().entrySet()) { origFieldName = e.getKey(); @@ -101,28 +95,22 @@ public EventBase transform(Entry entry) { } for (FieldValue fv : e.getValue()) { - AccessExpression fvExpression = AccessExpressionUtil.toAccessExpression(fv.getVisibility()); - expressionsToMerge.add(fvExpression); - try { - Markings fieldMarkings = this.markingFunctions.fromAccessExpression(fvExpression); - String value = new String(fv.getValue(), StandardCharsets.UTF_8); - // if this is a content field name, then replace the value with the uid - if (getContentFieldNames().contains(fieldName)) { - value = baseUid; - } - values.add(this.makeField(fieldName, fieldMarkings, fvExpression.getExpression(), entry.getKey().getTimestamp(), value)); - } catch (Exception e1) { - throw new RuntimeException("could not make markings from: " + fvExpression); + Markings fieldMarkings = AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(fv.getVisibility())); + markingsToMerge.add(fieldMarkings); + String value = new String(fv.getValue(), StandardCharsets.UTF_8); + // if this is a content field name, then replace the value with the uid + if (getContentFieldNames().contains(fieldName)) { + value = baseUid; } + values.add(this.makeField(fieldName, fieldMarkings, AccessExpressionUtil.toAccessExpression(fieldMarkings).getExpression(), + entry.getKey().getTimestamp(), value)); } } - AccessExpression combinedExpression = null; try { - combinedExpression = this.markingFunctions.combine(expressionsToMerge); - event.setMarkings(this.markingFunctions.fromAccessExpression(combinedExpression)); + event.setMarkings(AccessExpressionUtil.combine(this.markingFunctions, markingsToMerge)); } catch (Exception e1) { - throw new RuntimeException("could not make markings from: " + combinedExpression); + throw new RuntimeException("could not combine markings"); } event.setFields(new ArrayList<>(values)); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java index 037bb1b99ef..02e770edce4 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java @@ -8,7 +8,6 @@ import java.util.Map.Entry; import java.util.Set; -import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.ColumnVisibility; @@ -22,6 +21,7 @@ import datawave.core.query.logic.BaseQueryLogic; import datawave.data.type.StringType; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.microservice.query.Query; import datawave.query.attributes.Attribute; import datawave.query.attributes.Attributes; @@ -128,11 +128,8 @@ protected Collection buildFacets(Key documentKey, String f FieldCardinalityBase fc = this.responseObjectFactory.getFieldCardinality(); fc.setField(v.getFieldName()); - fc.setMarkings(markingFunctions.fromAccessExpression(AccessExpressionUtil.toAccessExpression(attr.getColumnVisibility()))); // reduces - // colvis based - // on - // visibility - fc.setColumnVisibility(MarkingFunctions.normalize(markingFunctions.toAccessExpression(fc.getMarkings())).getExpression()); + fc.setMarkings(AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(attr.getColumnVisibility()))); + fc.setColumnVisibility(AccessExpressionUtil.normalize(AccessExpressionUtil.toAccessExpression(fc.getMarkings())).getExpression()); fc.setLower(v.getFloorValue()); fc.setUpper(v.getCeilingValue()); fc.setCardinality(v.getEstimate().cardinality()); @@ -156,7 +153,7 @@ protected FacetsBase buildResponse(Document document, Key documentKey, ColumnVis final Collection documentFields = buildFacets(documentKey, null, document, eventCV, mf); - facetedResponse.setMarkings(mf.fromAccessExpression(AccessExpressionUtil.toAccessExpression(eventCV))); + facetedResponse.setMarkings(AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(eventCV))); facetedResponse.setFields(new ArrayList<>(documentFields)); // assign an estimate of the event size based on the document size @@ -170,7 +167,7 @@ protected FacetsBase buildResponse(Document document, Key documentKey, ColumnVis @Override public BaseQueryResponse createResponse(List resultList) { FacetQueryResponseBase response = responseObjectFactory.getFacetQueryResponse(); - Set combinedExpressions = new HashSet<>(); + Set> combinedMarkings = new HashSet<>(); for (Object result : resultList) { FacetsBase facet = (FacetsBase) result; @@ -181,15 +178,14 @@ public BaseQueryResponse createResponse(List resultList) { for (FieldCardinalityBase fcb : facet.getFields()) { if (StringUtils.isNotBlank(fcb.getColumnVisibility())) { - combinedExpressions.add(AccessExpressionUtil.toAccessExpression(new ColumnVisibility(fcb.getColumnVisibility()))); + combinedMarkings.add(AccessExpressionUtil + .fromAccessExpression(AccessExpressionUtil.toAccessExpression(new ColumnVisibility(fcb.getColumnVisibility())))); } } } try { - AccessExpression combinedExpression = this.markingFunctions.combine(combinedExpressions); - response.setMarkings(this.markingFunctions.fromAccessExpression(combinedExpression)); - + response.setMarkings(AccessExpressionUtil.combine(this.markingFunctions, combinedMarkings)); } catch (MarkingFunctions.Exception e) { log.warn(e); // original ignored these exceptions diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/FieldIndexCountQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/FieldIndexCountQueryTransformer.java index 823fc635187..e5a363d1069 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/FieldIndexCountQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/FieldIndexCountQueryTransformer.java @@ -61,13 +61,7 @@ public EventBase transform(Entry entry) { String key = entry.getKey(); Tuple val = entry.getValue(); - Markings markings = null; - try { - markings = this.markingFunctions.fromAccessExpression(AccessExpressionUtil.toAccessExpression(val.getColumnVisibility())); - } catch (Exception e) { - log.error("could not translate " + val.getColumnVisibility() + " to markings, skipping entry"); - return null; - } + Markings markings = AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(val.getColumnVisibility())); if (null == markings || markings.isEmpty()) { // can't process this one because we did not have valid security markings log.error("Transformer visibility interpreter was null, skipping entry"); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/GroupingDocumentTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/GroupingDocumentTransformer.java index 1c71976a125..9105545637f 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/GroupingDocumentTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/GroupingDocumentTransformer.java @@ -132,7 +132,7 @@ private Multimap getFieldToFieldWithGroupingContextMap(Collection FieldBase created = null; try { created = this.makeField(shortName, - this.markingFunctions.fromAccessExpression( + AccessExpressionUtil.fromAccessExpression( AccessExpressionUtil.toAccessExpression(new ColumnVisibility(field.getColumnVisibility()))), field.getColumnVisibility(), 0L, field.getValueOfTypedValue()); } catch (Exception ex) { diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/ShardIndexQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/ShardIndexQueryTransformer.java index d6aafa5ae0e..71f7e755ea4 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/ShardIndexQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/ShardIndexQueryTransformer.java @@ -20,7 +20,6 @@ import datawave.core.query.logic.BaseQueryLogicTransformer; import datawave.ingest.protobuf.Uid; import datawave.marking.MarkingFunctions; -import datawave.marking.MarkingFunctions.Exception; import datawave.marking.Markings; import datawave.microservice.query.Query; import datawave.query.model.QueryModel; @@ -68,12 +67,7 @@ public EventBase transform(Entry input) { EventBase event = responseObjectFactory.getEvent(); ColumnVisibility columnVisibility = new ColumnVisibility(entry.getKey().getColumnVisibility()); - Markings markings; - try { - markings = this.markingFunctions.fromAccessExpression(AccessExpressionUtil.toAccessExpression(columnVisibility)); - } catch (Exception e1) { - throw new RuntimeException("could not make markings from: " + columnVisibility); - } + Markings markings = AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(columnVisibility)); event.setMarkings(markings); List fields = new ArrayList<>(); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/ShardQueryCountTableTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/ShardQueryCountTableTransformer.java index a9b63b331ea..2e24ece9ea0 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/ShardQueryCountTableTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/ShardQueryCountTableTransformer.java @@ -13,7 +13,6 @@ import datawave.core.query.cachedresults.CacheableLogic; import datawave.core.query.logic.BaseQueryLogicTransformer; import datawave.marking.MarkingFunctions; -import datawave.marking.MarkingFunctions.Exception; import datawave.marking.Markings; import datawave.microservice.query.Query; import datawave.query.Constants; @@ -48,12 +47,7 @@ public EventBase transform(Entry untypedEntry) { Long count = untypedEntry.getKey(); ColumnVisibility vis = untypedEntry.getValue(); - Markings markings; - try { - markings = markingFunctions.fromAccessExpression(AccessExpressionUtil.toAccessExpression(vis)); - } catch (Exception e1) { - throw new IllegalArgumentException("Unable to translate markings", e1); - } + Markings markings = AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(vis)); EventBase e = this.responseObjectFactory.getEvent(); e.setMarkings(markings); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/TagCloudTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/TagCloudTransformer.java index 351df1c5b3f..ef0b1bfc1a7 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/TagCloudTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/TagCloudTransformer.java @@ -13,6 +13,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import datawave.core.common.util.AccessExpressionUtil; import datawave.core.query.logic.BaseQueryLogicTransformer; import datawave.marking.MarkingFunctions; import datawave.microservice.query.Query; @@ -203,11 +204,7 @@ private void configureV2TagCloud(TagCloudBase base, TagCloud tagCloud) { @SuppressWarnings("unchecked") private void convertAndSetMarkings(TagCloudBase base, AccessExpression visibility) { if (visibility != null && !visibility.getExpression().isEmpty()) { - try { - base.setMarkings(markingFunctions.fromAccessExpression(visibility)); - } catch (MarkingFunctions.Exception e) { - log.warn("Could not convert tag cloud visibility to markings: {}", visibility.getExpression(), e); - } + base.setMarkings(AccessExpressionUtil.fromAccessExpression(visibility)); } } diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/content/ContentQueryLogicTest.java b/warehouse/query-core/src/test/java/datawave/query/tables/content/ContentQueryLogicTest.java index f86be83688b..35ec03c1f83 100644 --- a/warehouse/query-core/src/test/java/datawave/query/tables/content/ContentQueryLogicTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/tables/content/ContentQueryLogicTest.java @@ -171,7 +171,7 @@ public void testDecodeViewParam() throws Exception { Value viewValue = new Value(Base64.encodeBytes("my happy message".getBytes())); Map.Entry entry = new AbstractMap.SimpleImmutableEntry<>(dataKey, viewValue); - markingFunctions.fromAccessExpression(new ColumnVisibility("A"), auths); + markingFunctions.translateFromColumnVisibilityForAuths(new ColumnVisibility("A"), auths); // test with decode view logic.initialize(mockClient, settings, Set.of(auths)); From 6fe36bfbc8d5803444552f63de399a6aa95d8631 Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Thu, 26 Mar 2026 12:54:06 -0400 Subject: [PATCH 11/63] cleanup --- .../marking/ColumnVisibilitySecurityMarking.java | 6 ++++-- .../query/cachedresults/CacheableQueryRowImpl.java | 4 ++-- .../cachedresults/CacheableQueryRowReader.java | 8 +++++--- .../core/common/util/AccessExpressionUtil.java | 11 ++++++----- .../modification/MutableMetadataHandler.java | 8 +++++--- .../datawave/marking/AccessExpressionMarkings.java | 3 ++- .../java/datawave/marking/MarkingFunctions.java | 5 +++-- .../MarkingFunctionsDefaultCombineTest.java | 3 ++- .../iterators/FrequencyMetadataAggregator.java | 2 +- .../webservice/response/objects/DefaultKey.java | 3 ++- .../webservice/response/LookupResponseTest.java | 3 ++- .../microservice/accumulo/admin/AdminService.java | 3 ++- .../accumulo/lookup/LookupService.java | 3 ++- .../dictionary/data/DefaultDescription.java | 4 +++- .../dictionary/DataDictionaryControllerLogic.java | 6 ++++-- .../metadata/DefaultMetadataFieldScanner.java | 6 ++++-- .../metadata/MetadataDescriptionsHelper.java | 14 ++++++-------- .../microservice/querymetric/BaseQueryMetric.java | 3 ++- .../microservice/querymetric/QueryMetricTest.java | 7 ++++--- .../querymetric/QueryMetricTestBase.java | 3 ++- .../ingest/config/RawRecordContainerImpl.java | 3 ++- .../ingest/data/config/MarkingsHelper.java | 3 ++- .../datawave/ingest/mapreduce/SimpleRawRecord.java | 3 ++- .../ingest/csv/mr/input/CSVRecordReader.java | 4 +++- .../util/keyword/DefaultTagCloudUtils.java | 4 +++- .../main/java/datawave/util/keyword/TagCloud.java | 4 ++-- .../FieldIndexCountingIteratorPerVisibility.java | 2 +- .../iterators/GlobalIndexDateSummaryIterator.java | 2 +- .../core/iterators/ResultCountingIterator.java | 2 +- .../datawave/query/attributes/AttributeBag.java | 2 +- .../java/datawave/query/attributes/Document.java | 2 +- .../query/common/grouping/GroupingUtils.java | 2 +- .../query/discovery/DiscoveryIterator.java | 2 +- .../query/discovery/DiscoveryTransformer.java | 4 ++-- .../query/discovery/TermInfoAggregation.java | 4 ++-- .../iterators/FieldIndexCountingIterator.java | 2 +- .../datawave/query/jexl/DatawaveInterpreter.java | 2 +- .../query/predicate/ValueToAttributes.java | 2 +- .../query/table/parser/ContentKeyValueFactory.java | 6 ++---- .../query/table/parser/EventKeyValueFactory.java | 7 +++---- .../query/table/parser/KeywordKeyValueFactory.java | 2 +- .../table/parser/TermFrequencyKeyValueFactory.java | 5 ++--- .../tables/shard/CountAggregatingIterator.java | 2 +- .../tables/shard/CountResultPostprocessor.java | 2 +- .../tables/shard/FieldIndexCountQueryLogic.java | 2 +- .../query/transformer/ContentQueryTransformer.java | 7 +------ .../query/transformer/DocumentTransformer.java | 5 ++--- .../transformer/DocumentTransformerSupport.java | 5 +++-- .../query/transformer/EdgeQueryTransformer.java | 2 +- .../query/transformer/EventQueryTransformer.java | 4 ++-- .../query/transformer/FacetedTransformer.java | 9 ++++----- .../FieldIndexCountQueryTransformer.java | 2 +- .../transformer/GroupingDocumentTransformer.java | 2 +- .../transformer/ShardIndexQueryTransformer.java | 2 +- .../ShardQueryCountTableTransformer.java | 2 +- .../query/transformer/TagCloudTransformer.java | 2 +- .../webservice/query/result/event/SimpleField.java | 4 +++- .../query/common/grouping/DocumentGrouperTest.java | 3 ++- .../query/tables/keyword/TagCloudTestUtil.java | 3 ++- .../transformer/FieldRenameTransformTest.java | 8 ++++---- 60 files changed, 131 insertions(+), 109 deletions(-) diff --git a/core/base-rest-responses/src/main/java/datawave/marking/ColumnVisibilitySecurityMarking.java b/core/base-rest-responses/src/main/java/datawave/marking/ColumnVisibilitySecurityMarking.java index c2f456918a5..c3639140978 100644 --- a/core/base-rest-responses/src/main/java/datawave/marking/ColumnVisibilitySecurityMarking.java +++ b/core/base-rest-responses/src/main/java/datawave/marking/ColumnVisibilitySecurityMarking.java @@ -13,6 +13,8 @@ import org.apache.accumulo.access.AccessExpression; import com.google.common.base.Preconditions; +import org.apache.accumulo.access.examples.ParseExamples; +import org.apache.accumulo.access.impl.AccessExpressionImpl; @XmlAccessorType(XmlAccessType.NONE) @XmlAccessorOrder(XmlAccessOrder.ALPHABETICAL) @@ -20,7 +22,7 @@ public class ColumnVisibilitySecurityMarking implements SecurityMarking { public static final String VISIBILITY_MARKING = "columnVisibility"; - private static final Access ACCESS = Access.builder().build(); + private static final Access ACCESS = ParseExamples.ACCESS; @XmlAttribute(name = "columnVisibility") private String columnVisibility = null; @@ -49,7 +51,7 @@ public void setColumnVisibility(String columnVisibility) { @Override public AccessExpression toAccessExpression() { if (columnVisibility == null || columnVisibility.isEmpty()) { - return ACCESS.newExpression(""); + return AccessExpressionImpl.EMPTY; } return ACCESS.newExpression(columnVisibility); } diff --git a/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java b/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java index c11ef3da668..27dde06fe60 100644 --- a/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java +++ b/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java @@ -9,6 +9,7 @@ import java.util.Set; import java.util.TreeSet; +import org.apache.accumulo.access.impl.AccessExpressionImpl; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,8 +28,7 @@ public class CacheableQueryRowImpl extends CacheableQueryRow implements ObjectSizeOf { private static final Logger LOGGER = LoggerFactory.getLogger(CacheableQueryRowImpl.class); - private static final Markings EMPTY_MARKINGS = AccessExpressionMarkings.builder().accessExpression(MarkingFunctions.Default.ACCESS.newExpression("")) - .build(); + private static final Markings EMPTY_MARKINGS = AccessExpressionMarkings.builder().accessExpression(AccessExpressionImpl.EMPTY).build(); private String user = null; private String queryId = null; diff --git a/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowReader.java b/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowReader.java index c03388213f0..0f9fd6e0646 100644 --- a/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowReader.java +++ b/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowReader.java @@ -18,6 +18,8 @@ import datawave.webservice.query.cachedresults.CacheableQueryRow; import datawave.webservice.query.result.event.ResponseObjectFactory; +import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; + public class CacheableQueryRowReader { private static Logger log = LoggerFactory.getLogger(CacheableQueryRowReader.class); @@ -90,7 +92,7 @@ public static CacheableQueryRow createRow(ResultSet cachedRowSet, Set fi if (columnToIndexMap.get("_markings_") != null) { String mStr = cachedRowSet.getString(columnToIndexMap.get("_markings_")); if (mStr != null && !mStr.isEmpty()) { - cqfc.setMarkings(AccessExpressionUtil.fromAccessExpression(MarkingFunctions.Default.ACCESS.newExpression(mStr))); + cqfc.setMarkings(AccessExpressionUtil.toAccessExpressionMarkings(ACCESS.newExpression(mStr))); } } if (columnToIndexMap.get("_column_markings_") != null) { @@ -104,11 +106,11 @@ public static CacheableQueryRow createRow(ResultSet cachedRowSet, Set fi int x = combinedString.lastIndexOf(':'); if (x >= 0) { String markingStr = combinedString.substring(0, x); - columnMarkingsMap.put(columnName, AccessExpressionUtil.fromAccessExpression(MarkingFunctions.Default.ACCESS.newExpression(markingStr))); + columnMarkingsMap.put(columnName, AccessExpressionUtil.toAccessExpressionMarkings(ACCESS.newExpression(markingStr))); columnVisibilityMap.put(columnName, combinedString.substring(x + 1)); } else { columnMarkingsMap.put(columnName, - AccessExpressionUtil.fromAccessExpression(MarkingFunctions.Default.ACCESS.newExpression(combinedString))); + AccessExpressionUtil.toAccessExpressionMarkings(ACCESS.newExpression(combinedString))); columnVisibilityMap.put(columnName, ""); } } diff --git a/core/common-util/src/main/java/datawave/core/common/util/AccessExpressionUtil.java b/core/common-util/src/main/java/datawave/core/common/util/AccessExpressionUtil.java index 298b4aca0f5..277afbca541 100644 --- a/core/common-util/src/main/java/datawave/core/common/util/AccessExpressionUtil.java +++ b/core/common-util/src/main/java/datawave/core/common/util/AccessExpressionUtil.java @@ -8,6 +8,7 @@ import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.access.ParsedAccessExpression; import org.apache.accumulo.access.examples.ParseExamples; +import org.apache.accumulo.access.impl.AccessExpressionImpl; import org.apache.accumulo.core.security.ColumnVisibility; import datawave.marking.AccessExpressionMarkings; @@ -22,9 +23,9 @@ */ public final class AccessExpressionUtil { - private static final Access ACCESS = Access.builder().build(); + private static final Access ACCESS = ParseExamples.ACCESS; - private static final AccessExpression EMPTY_EXPRESSION = ACCESS.newExpression(""); + private static final AccessExpression EMPTY_EXPRESSION = AccessExpressionImpl.EMPTY; private AccessExpressionUtil() {} @@ -61,7 +62,7 @@ public static AccessExpression toAccessExpression(Markings markings) { * The AccessExpression to embed in the wrapper class * @return the wrapped access expression */ - public static AccessExpressionMarkings fromAccessExpression(AccessExpression expression) { + public static AccessExpressionMarkings toAccessExpressionMarkings(AccessExpression expression) { return AccessExpressionMarkings.builder().accessExpression(expression).build(); } @@ -125,8 +126,8 @@ public static AccessExpression normalize(AccessExpression expression) { if (expr.isEmpty()) { return expression; } - ParsedAccessExpression parsed = MarkingFunctions.Default.ACCESS.newParsedExpression(expr); - return MarkingFunctions.Default.ACCESS.newExpression(ParseExamples.normalize(parsed).expression); + ParsedAccessExpression parsed = ACCESS.newParsedExpression(expr); + return ACCESS.newExpression(ParseExamples.normalize(parsed).expression); } /** diff --git a/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java b/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java index c4fd81680fd..cb8c8bfa8ea 100644 --- a/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java +++ b/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java @@ -76,6 +76,8 @@ import datawave.webservice.result.EventQueryResponseBase; import datawave.webservice.result.GenericResponse; +import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; + /** * Class that handles requests for modification requests (INSERT, UPDATE, DELETE) for metadata in the shard schema.
*
@@ -315,7 +317,7 @@ public void process(AccumuloClient client, ModificationRequestBase request, Map< /** * Convert legacy Map<String,String> field markings (from DefaultModificationRequest) to Markings<?>. */ - private Markings convertFieldMarkings(Map rawMarkings) throws MarkingFunctions.Exception { + private Markings convertFieldMarkings(Map rawMarkings) { if (rawMarkings == null || rawMarkings.isEmpty()) { return null; } @@ -323,7 +325,7 @@ private Markings convertFieldMarkings(Map rawMarkings) throws if (vis.isEmpty()) { return null; } - return AccessExpressionUtil.fromAccessExpression(MarkingFunctions.Default.ACCESS.newExpression(vis)); + return AccessExpressionUtil.toAccessExpressionMarkings(ACCESS.newExpression(vis)); } public void process(AccumuloClient client, ModificationRequestBase request, Map> mutableFieldList, Set userAuths, @@ -871,7 +873,7 @@ protected List> getField(AccumuloClient client, Set markings = AccessExpressionUtil - .fromAccessExpression(AccessExpressionUtil.toAccessExpression(e.getKey().getColumnVisibilityParsed())); + .toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(e.getKey().getColumnVisibilityParsed())); if (null != oldFieldMarkings && !oldFieldMarkings.equals(markings)) { log.trace("Skipping key that does not match with markings: {}", e.getKey()); continue; diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java index 99438ffc453..9458a9b6599 100644 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java @@ -9,13 +9,14 @@ import lombok.Builder; import lombok.Data; +import org.apache.accumulo.access.examples.ParseExamples; @Data @Builder @JsonAutoDetect(getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE) public class AccessExpressionMarkings implements Markings { - private static final Access ACCESS = Access.builder().build(); + private static final Access ACCESS = ParseExamples.ACCESS; public static final String COLUMN_VISIBILITY_KEY = "columnVisibility"; private final AccessExpression accessExpression; diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java index 23a8fd538a8..b26bea712af 100644 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java @@ -9,6 +9,7 @@ import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.access.ParsedAccessExpression; import org.apache.accumulo.access.examples.ParseExamples; +import org.apache.accumulo.access.impl.AccessExpressionImpl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.support.ClassPathXmlApplicationContext; @@ -48,8 +49,8 @@ public Exception(Throwable cause) { } class Default implements MarkingFunctions { - public static final Access ACCESS = Access.builder().build(); - public static final AccessExpressionMarkings EMPTY_MARKINGS = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression("")).build(); + private static final Access ACCESS = ParseExamples.ACCESS; + private static final AccessExpressionMarkings EMPTY_MARKINGS = AccessExpressionMarkings.builder().accessExpression(AccessExpressionImpl.EMPTY).build(); @Override public AccessExpressionMarkings combine(Collection markings) { diff --git a/core/utils/accumulo-utils/src/test/java/datawave/marking/MarkingFunctionsDefaultCombineTest.java b/core/utils/accumulo-utils/src/test/java/datawave/marking/MarkingFunctionsDefaultCombineTest.java index 54f5230d4df..2ff91e3f71c 100644 --- a/core/utils/accumulo-utils/src/test/java/datawave/marking/MarkingFunctionsDefaultCombineTest.java +++ b/core/utils/accumulo-utils/src/test/java/datawave/marking/MarkingFunctionsDefaultCombineTest.java @@ -5,10 +5,11 @@ import java.util.List; import org.apache.accumulo.access.Access; +import org.apache.accumulo.access.examples.ParseExamples; import org.junit.jupiter.api.Test; public class MarkingFunctionsDefaultCombineTest { - public static final Access ACCESS = Access.builder().build(); + public static final Access ACCESS = ParseExamples.ACCESS; @Test public void testCombineAnds() throws MarkingFunctions.Exception { diff --git a/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java b/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java index 156535fc1f2..494d1646e66 100644 --- a/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java +++ b/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java @@ -587,7 +587,7 @@ private ColumnVisibility combineAllVisibilities() { Set visibilities = visibilityToDateFrequencies.keySet(); try { return AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(markingFunctions, visibilities.stream() - .map(cv -> AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(cv))).collect(Collectors.toSet()))); + .map(cv -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(cv))).collect(Collectors.toSet()))); } catch (MarkingFunctions.Exception e) { log.error("Failed to combine visibilities " + visibilities); throw new IllegalArgumentException("Failed to combine visibilities " + visibilities, e); diff --git a/microservices/services/accumulo/api/src/main/java/datawave/webservice/response/objects/DefaultKey.java b/microservices/services/accumulo/api/src/main/java/datawave/webservice/response/objects/DefaultKey.java index d4063faa0a7..a4b00e69223 100644 --- a/microservices/services/accumulo/api/src/main/java/datawave/webservice/response/objects/DefaultKey.java +++ b/microservices/services/accumulo/api/src/main/java/datawave/webservice/response/objects/DefaultKey.java @@ -1,6 +1,7 @@ package datawave.webservice.response.objects; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -114,7 +115,7 @@ public void setColQual(String colQual) { public void setColumnVisibility(String colVis) { this.columnVisibility = (colVis == null) ? new TypedValue("") : new TypedValue(colVis); String expr = this.columnVisibility.getValue().toString(); - this.markings = AccessExpressionMarkings.builder().accessExpression(MarkingFunctions.Default.ACCESS.newExpression(expr)).build(); + this.markings = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression(expr)).build(); } public void setTimestamp(long timestamp) { diff --git a/microservices/services/accumulo/api/src/test/java/datawave/webservice/response/LookupResponseTest.java b/microservices/services/accumulo/api/src/test/java/datawave/webservice/response/LookupResponseTest.java index c0b7c973218..75bcf85d522 100644 --- a/microservices/services/accumulo/api/src/test/java/datawave/webservice/response/LookupResponseTest.java +++ b/microservices/services/accumulo/api/src/test/java/datawave/webservice/response/LookupResponseTest.java @@ -1,5 +1,6 @@ package datawave.webservice.response; +import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -39,7 +40,7 @@ private void responseSetup() { lookupResponse = new LookupResponse(); List entryList = new ArrayList<>(); for (int i = 0; i < 3; i++) { - AccessExpressionMarkings markings = AccessExpressionMarkings.builder().accessExpression(MarkingFunctions.Default.ACCESS.newExpression("A&B&C" + i)) + AccessExpressionMarkings markings = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression("A&B&C" + i)) .build(); KeyBase responseKey = new DefaultKey(); responseKey.setRow("row" + i); diff --git a/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/admin/AdminService.java b/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/admin/AdminService.java index 75537241973..d1ad8f8b6e9 100644 --- a/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/admin/AdminService.java +++ b/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/admin/AdminService.java @@ -1,6 +1,7 @@ package datawave.microservice.accumulo.admin; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; import java.util.ArrayList; import java.util.HashMap; @@ -571,7 +572,7 @@ public ValidateVisibilityResponse validateVisibilities(String[] visibilityArray) vis.setValid(false); visibilityList.add(vis); try { - Markings markings = AccessExpressionUtil.fromAccessExpression(MarkingFunctions.Default.ACCESS.newExpression(v)); + Markings markings = AccessExpressionUtil.toAccessExpressionMarkings(ACCESS.newExpression(v)); vis.setVisibility(v); vis.setValid(true); vis.setMarkings(markings); diff --git a/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/lookup/LookupService.java b/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/lookup/LookupService.java index 82f263f2bbc..67f73e63460 100644 --- a/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/lookup/LookupService.java +++ b/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/lookup/LookupService.java @@ -1,6 +1,7 @@ package datawave.microservice.accumulo.lookup; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; import java.util.ArrayList; import java.util.Arrays; @@ -220,7 +221,7 @@ public LookupResponse lookup(LookupRequest request, DatawaveUserDetails currentU } final Markings markings = AccessExpressionUtil - .fromAccessExpression(MarkingFunctions.Default.ACCESS.newExpression(k.getColumnVisibility().toString())); + .toAccessExpressionMarkings(ACCESS.newExpression(k.getColumnVisibility().toString())); final KeyBase responseKey = responseObjectFactory.createKey(); responseKey.setRow(currRow); diff --git a/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java b/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java index 91cbe9679fb..e187caeab26 100644 --- a/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java +++ b/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java @@ -23,6 +23,8 @@ import io.protostuff.Output; import io.protostuff.Schema; +import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; + @XmlAccessorType(XmlAccessType.NONE) @XmlAccessorOrder(XmlAccessOrder.ALPHABETICAL) public class DefaultDescription extends DescriptionBase implements Serializable, Message { @@ -66,7 +68,7 @@ public void setMarkings(Markings markings) { public Markings getMarkings() { if (this.markingsMap != null && this.markingsMap.containsKey(CV_KEY)) { String expr = this.markingsMap.get(CV_KEY); - return AccessExpressionMarkings.builder().accessExpression(MarkingFunctions.Default.ACCESS.newExpression(expr)).build(); + return AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression(expr)).build(); } return null; } diff --git a/microservices/services/dictionary/service/src/main/java/datawave/microservice/dictionary/DataDictionaryControllerLogic.java b/microservices/services/dictionary/service/src/main/java/datawave/microservice/dictionary/DataDictionaryControllerLogic.java index 0f378bd26d6..47bb8d13484 100644 --- a/microservices/services/dictionary/service/src/main/java/datawave/microservice/dictionary/DataDictionaryControllerLogic.java +++ b/microservices/services/dictionary/service/src/main/java/datawave/microservice/dictionary/DataDictionaryControllerLogic.java @@ -31,6 +31,8 @@ import datawave.webservice.metadata.MetadataFieldBase; import datawave.webservice.result.VoidResponse; +import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; + public class DataDictionaryControllerLogic,DICT extends DataDictionaryBase,META extends MetadataFieldBase,FIELD extends DictionaryFieldBase,FIELDS extends FieldsBase> { private final DataDictionaryProperties dataDictionaryConfiguration; private final DataDictionary dataDictionary; @@ -170,7 +172,7 @@ public VoidResponse setDescriptionPut(String fieldName, String datatype, String public VoidResponse setDescriptionPost(String fieldName, String datatype, String description, String modelName, String modelTable, String columnVisibility, DatawaveUserDetails currentUser) throws Exception { DESC desc = this.responseObjectFactory.getDescription(); - Markings markings = AccessExpressionMarkings.builder().accessExpression(MarkingFunctions.Default.ACCESS.newExpression(columnVisibility)).build(); + Markings markings = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression(columnVisibility)).build(); desc.setMarkings(markings); desc.setDescription(description); @@ -283,7 +285,7 @@ public FIELDS fieldNameDescription(String fieldName, String datatype, String mod */ public VoidResponse deleteDescription(String fieldName, String datatype, String modelName, String modelTable, String columnVisibility, DatawaveUserDetails currentUser) throws Exception { - Markings markings = AccessExpressionMarkings.builder().accessExpression(MarkingFunctions.Default.ACCESS.newExpression(columnVisibility)).build(); + Markings markings = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression(columnVisibility)).build(); DESC desc = this.responseObjectFactory.getDescription(); desc.setDescription(""); desc.setMarkings(markings); diff --git a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java index cce616eb3b2..2932f2140f2 100644 --- a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java +++ b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java @@ -34,6 +34,8 @@ import datawave.webservice.dictionary.data.DefaultFields; import datawave.webservice.metadata.DefaultMetadataField; +import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; + public class DefaultMetadataFieldScanner { private static final Logger log = LoggerFactory.getLogger(DefaultMetadataFieldScanner.class); @@ -273,11 +275,11 @@ private void setFieldNameAndAlias() { } // Extract the description from the current value and add it to the current {@link DefaultMetadataField}. - private void setDescriptions() throws MarkingFunctions.Exception { + private void setDescriptions() { DefaultDescription description = responseObjectFactory.getDescription(); description.setDescription(currValue.toString()); description.setMarkings( - AccessExpressionUtil.fromAccessExpression(MarkingFunctions.Default.ACCESS.newExpression(currKey.getColumnVisibility().toString()))); + AccessExpressionUtil.toAccessExpressionMarkings(ACCESS.newExpression(currKey.getColumnVisibility().toString()))); currField.getDescriptions().add(description); } diff --git a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java index dcc40bd904b..80e2cc63682 100644 --- a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java +++ b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java @@ -3,10 +3,8 @@ import static java.nio.charset.StandardCharsets.UTF_8; import java.util.Collections; -import java.util.Map; import java.util.Map.Entry; import java.util.Set; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import org.apache.accumulo.core.client.AccumuloClient; @@ -88,7 +86,7 @@ public SetMultimap getDescriptions(Set ingestTypeFil } public SetMultimap getFieldDescriptions(Set ingestTypeFilter) - throws TableNotFoundException, ExecutionException, MarkingFunctions.Exception { + throws TableNotFoundException, MarkingFunctions.Exception { SetMultimap descriptions = getDescriptions(ingestTypeFilter); SetMultimap fieldDescriptions = HashMultimap.create(); @@ -121,7 +119,7 @@ public void setDescription(MetadataEntry entry, DescriptionBase desc) } public void setDescriptions(MetadataEntry entry, Set descs) - throws TableNotFoundException, MutationsRejectedException, MarkingFunctions.Exception { + throws TableNotFoundException, MutationsRejectedException { BatchWriter bw = null; try { BatchWriterConfig bwConfig = new BatchWriterConfig().setMaxMemory(10000L).setMaxLatency(100L, TimeUnit.MILLISECONDS).setMaxWriteThreads(1); @@ -156,7 +154,7 @@ public void setDescriptions(MetadataEntry entry, Set * if writing the update to Accumulo fails */ public void removeDescription(MetadataEntry entry, DescriptionBase desc) - throws TableNotFoundException, MutationsRejectedException, MarkingFunctions.Exception { + throws TableNotFoundException, MutationsRejectedException { BatchWriter bw = null; try { BatchWriterConfig bwConfig = new BatchWriterConfig().setMaxMemory(10000L).setMaxLatency(100L, TimeUnit.MILLISECONDS).setMaxWriteThreads(1); @@ -177,7 +175,7 @@ public void removeDescription(MetadataEntry entry, DescriptionBase desc) } - protected SetMultimap loadDescriptions() throws TableNotFoundException, MarkingFunctions.Exception { + protected SetMultimap loadDescriptions() throws TableNotFoundException { if (log.isTraceEnabled()) log.trace("loadDescriptions from table: " + metadataTableName); // unlike other entries, the desc colf entries have many auths set. We'll use the fullUserAuths in the scanner instead @@ -202,7 +200,7 @@ protected SetMultimap loadDescriptions() throws TableNotFoun } - private Markings getMarkings(Key k) throws MarkingFunctions.Exception { - return AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(k.getColumnVisibilityParsed())); + private Markings getMarkings(Key k) { + return AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(k.getColumnVisibilityParsed())); } } diff --git a/microservices/services/query-metric/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java b/microservices/services/query-metric/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java index dc78bb4d394..fec797334a1 100644 --- a/microservices/services/query-metric/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java +++ b/microservices/services/query-metric/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java @@ -26,6 +26,7 @@ import org.apache.accumulo.access.Access; import org.apache.accumulo.access.AccessExpression; +import org.apache.accumulo.access.examples.ParseExamples; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.builder.CompareToBuilder; import org.apache.commons.lang3.builder.EqualsBuilder; @@ -1167,7 +1168,7 @@ public void setPredictions(Set predictions) { this.predictions = predictions; } - private static final Access ACCESS = Access.builder().build(); + private static final Access ACCESS = ParseExamples.ACCESS; @Override public void setMarkings(Markings markings) { diff --git a/microservices/services/query-metric/api/src/test/java/datawave/microservice/querymetric/QueryMetricTest.java b/microservices/services/query-metric/api/src/test/java/datawave/microservice/querymetric/QueryMetricTest.java index d078b0979da..8b855965507 100644 --- a/microservices/services/query-metric/api/src/test/java/datawave/microservice/querymetric/QueryMetricTest.java +++ b/microservices/services/query-metric/api/src/test/java/datawave/microservice/querymetric/QueryMetricTest.java @@ -1,6 +1,7 @@ package datawave.microservice.querymetric; import static datawave.webservice.query.exception.DatawaveErrorCode.NO_QUERY_RESULTS_FOUND; +import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -51,7 +52,7 @@ public class QueryMetricTest { @BeforeAll public static void setup() { queryMetric = new QueryMetric(); - queryMetric.setMarkings(AccessExpressionMarkings.builder().accessExpression(MarkingFunctions.Default.ACCESS.newExpression("PUBLIC")).build()); + queryMetric.setMarkings(AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression("PUBLIC")).build()); negativeSelectors = new ArrayList<>(); negativeSelectors.add("negativeSelector1"); positiveSelectors = new ArrayList<>(); @@ -100,7 +101,7 @@ public void testSettersGetters() { queryMetric.setHost("host"); queryMetric.setLastUpdated(d); queryMetric.setLifecycle(Lifecycle.INITIALIZED); - queryMetric.setMarkings(AccessExpressionMarkings.builder().accessExpression(MarkingFunctions.Default.ACCESS.newExpression("PUBLIC")).build()); + queryMetric.setMarkings(AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression("PUBLIC")).build()); queryMetric.setNegativeSelectors(negativeSelectors); queryMetric.setNumUpdates(0); queryMetric.setPageTimes(pageTimes); @@ -230,7 +231,7 @@ public void testVersionSerialization() throws Exception { qm.setHost("host"); qm.setLastUpdated(d); qm.setLifecycle(BaseQueryMetric.Lifecycle.INITIALIZED); - qm.setMarkings(AccessExpressionMarkings.builder().accessExpression(MarkingFunctions.Default.ACCESS.newExpression("PUBLIC")).build()); + qm.setMarkings(AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression("PUBLIC")).build()); qm.setNegativeSelectors(negativeSelectors); qm.setNumUpdates(0); qm.setPageTimes(pageTimes); diff --git a/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java b/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java index e6d2f7ea944..c8bd2741f63 100644 --- a/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java +++ b/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java @@ -2,6 +2,7 @@ import static datawave.microservice.querymetric.config.HazelcastMetricCacheConfiguration.INCOMING_METRICS; import static datawave.security.authorization.DatawaveUser.UserType.USER; +import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -154,7 +155,7 @@ public class QueryMetricTestBase { protected AccumuloClient accumuloClient; static { - AccessExpression ae = MarkingFunctions.Default.ACCESS.newExpression("A&C"); + AccessExpression ae = ACCESS.newExpression("A&C"); metricMarkings = AccessExpressionMarkings.builder().accessExpression(ae).build(); } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/config/RawRecordContainerImpl.java b/warehouse/ingest-core/src/main/java/datawave/ingest/config/RawRecordContainerImpl.java index 23ad57d8da6..1d335f7ec2c 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/config/RawRecordContainerImpl.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/config/RawRecordContainerImpl.java @@ -17,6 +17,7 @@ import org.apache.accumulo.access.Access; import org.apache.accumulo.access.AccessExpression; +import org.apache.accumulo.access.examples.ParseExamples; import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.ToStringBuilder; @@ -82,7 +83,7 @@ public class RawRecordContainerImpl implements Writable, Configurable, RawRecord private Object auxData = null; private Map auxMap = null; - private static final Access ACCESS = Access.builder().build(); + private static final Access ACCESS = ParseExamples.ACCESS; // RawRecordContainer support Markings securityMarkings = null; diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/MarkingsHelper.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/MarkingsHelper.java index 48719633519..7558a088d2c 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/MarkingsHelper.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/MarkingsHelper.java @@ -5,6 +5,7 @@ import java.util.Map.Entry; import org.apache.accumulo.access.Access; +import org.apache.accumulo.access.examples.ParseExamples; import org.apache.hadoop.conf.Configuration; import datawave.ingest.data.Type; @@ -54,7 +55,7 @@ public interface MarkingsHelper { */ class NoOp implements MarkingsHelper { - private static final Access ACCESS = Access.builder().build(); + private static final Access ACCESS = ParseExamples.ACCESS; private Map> fieldMarkingMap = new HashMap<>(); private Markings defaultMarkings = null; diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/SimpleRawRecord.java b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/SimpleRawRecord.java index e48309db946..c3de6458b85 100644 --- a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/SimpleRawRecord.java +++ b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/SimpleRawRecord.java @@ -12,6 +12,7 @@ import org.apache.accumulo.access.Access; import org.apache.accumulo.access.AccessExpression; +import org.apache.accumulo.access.examples.ParseExamples; import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.hadoop.io.Writable; @@ -30,7 +31,7 @@ */ public class SimpleRawRecord implements RawRecordContainer, Writable { - private static final Access ACCESS = Access.builder().build(); + private static final Access ACCESS = ParseExamples.ACCESS; private UIDBuilder uidBuilder = HashUID.builder(); private Markings securityMarkings = null; diff --git a/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java b/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java index 14708dc40c1..50c0f8c0f8d 100644 --- a/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java +++ b/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java @@ -31,6 +31,8 @@ import datawave.marking.AccessExpressionMarkings; import datawave.marking.MarkingFunctions; +import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; + public class CSVRecordReader extends CSVReaderBase implements EventFixer { private static final Logger log = Logger.getLogger(CSVRecordReader.class); @@ -127,7 +129,7 @@ public RawRecordContainer getEvent() { protected void decorateEvent() { if (securityMarkingExpression != null) { try { - AccessExpression ae = AccessExpressionUtil.normalize(MarkingFunctions.Default.ACCESS.newExpression(securityMarkingExpression)); + AccessExpression ae = AccessExpressionUtil.normalize(ACCESS.newExpression(securityMarkingExpression)); event.setSecurityMarkings(AccessExpressionMarkings.builder().accessExpression(ae).build()); } catch (Exception e) { log.error("Could not set security markings for the event", e); diff --git a/warehouse/keyword-common/src/main/java/datawave/util/keyword/DefaultTagCloudUtils.java b/warehouse/keyword-common/src/main/java/datawave/util/keyword/DefaultTagCloudUtils.java index b0ff4c0e086..b3e8388a5bf 100644 --- a/warehouse/keyword-common/src/main/java/datawave/util/keyword/DefaultTagCloudUtils.java +++ b/warehouse/keyword-common/src/main/java/datawave/util/keyword/DefaultTagCloudUtils.java @@ -16,6 +16,8 @@ import datawave.marking.MarkingFunctions; import datawave.marking.Markings; +import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; + /** * Default implementations for pluggable utilities for generating tag clouds, includes mechanisms to partition keywords into separate tag clouds, combine or * merge visibility strings, and calculate scores, source collections and frequencies of individual keywords based on observed results @@ -29,7 +31,7 @@ public AccessExpression generateCombinedVisibility(Set visibilities) { if (visibilities == null || visibilities.isEmpty()) { return null; } - Set> markings = visibilities.stream().map(v -> AccessExpressionUtil.fromAccessExpression(MarkingFunctions.Default.ACCESS.newExpression(v))) + Set> markings = visibilities.stream().map(v -> AccessExpressionUtil.toAccessExpressionMarkings(ACCESS.newExpression(v))) .collect(Collectors.toSet()); try { return AccessExpressionUtil.toAccessExpression(AccessExpressionUtil.combine(new MarkingFunctions.Default(), markings)); diff --git a/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloud.java b/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloud.java index 8c338f2b7e0..5bcd58871ad 100644 --- a/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloud.java +++ b/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloud.java @@ -16,7 +16,7 @@ import com.google.gson.Gson; -import datawave.marking.MarkingFunctions; +import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; /** A tag cloud - a collection of tags that have a keyword, score, frequency and list of sources from which they originated */ public class TagCloud { @@ -57,7 +57,7 @@ public AccessExpression getVisibility() { if (visibility == null || visibility.isEmpty()) { return null; } - return MarkingFunctions.Default.ACCESS.newExpression(visibility); + return ACCESS.newExpression(visibility); } public Collection getResults() { diff --git a/warehouse/query-core/src/main/java/datawave/core/iterators/FieldIndexCountingIteratorPerVisibility.java b/warehouse/query-core/src/main/java/datawave/core/iterators/FieldIndexCountingIteratorPerVisibility.java index d0414d7b1ea..4369144d98c 100644 --- a/warehouse/query-core/src/main/java/datawave/core/iterators/FieldIndexCountingIteratorPerVisibility.java +++ b/warehouse/query-core/src/main/java/datawave/core/iterators/FieldIndexCountingIteratorPerVisibility.java @@ -568,7 +568,7 @@ private Map buildReturnKeys() { try { // Calculate the columnVisibility for this key from the combiner. cv = AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(MarkingFunctions.Factory.createMarkingFunctions(), - columnVisibilities.stream().map(c -> AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(c))) + columnVisibilities.stream().map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))) .collect(Collectors.toSet()))); } catch (Exception e) { log.error("Could not create combined columnVisibility for the count", e); diff --git a/warehouse/query-core/src/main/java/datawave/core/iterators/GlobalIndexDateSummaryIterator.java b/warehouse/query-core/src/main/java/datawave/core/iterators/GlobalIndexDateSummaryIterator.java index 3c72d342d6e..f5615e03bb9 100644 --- a/warehouse/query-core/src/main/java/datawave/core/iterators/GlobalIndexDateSummaryIterator.java +++ b/warehouse/query-core/src/main/java/datawave/core/iterators/GlobalIndexDateSummaryIterator.java @@ -260,7 +260,7 @@ public Map getKeyValues() throws IOException { // Note that the access controls found in the combined ColumnVisibility will be pulled out appropriately here ColumnVisibility cv = AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine( MarkingFunctions.Factory.createMarkingFunctions(), - columnVisibilities.stream().map(c -> AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(c))) + columnVisibilities.stream().map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))) .collect(Collectors.toSet()))); // Create a new Key compatible with the shardIndex key format diff --git a/warehouse/query-core/src/main/java/datawave/core/iterators/ResultCountingIterator.java b/warehouse/query-core/src/main/java/datawave/core/iterators/ResultCountingIterator.java index 81e6e5fd237..3a92a617593 100644 --- a/warehouse/query-core/src/main/java/datawave/core/iterators/ResultCountingIterator.java +++ b/warehouse/query-core/src/main/java/datawave/core/iterators/ResultCountingIterator.java @@ -212,7 +212,7 @@ public Value getTopValue() { try { cv = AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(MarkingFunctions.Factory.createMarkingFunctions(), - columnVisibilities.stream().map(c -> AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(c))) + columnVisibilities.stream().map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))) .collect(Collectors.toSet()))); } catch (MarkingFunctions.Exception e) { log.error("Could not create combined columnVisibility for the count", e); diff --git a/warehouse/query-core/src/main/java/datawave/query/attributes/AttributeBag.java b/warehouse/query-core/src/main/java/datawave/query/attributes/AttributeBag.java index baf05bf2811..2a9b1846e89 100644 --- a/warehouse/query-core/src/main/java/datawave/query/attributes/AttributeBag.java +++ b/warehouse/query-core/src/main/java/datawave/query/attributes/AttributeBag.java @@ -82,7 +82,7 @@ protected ColumnVisibility combineAndSetColumnVisibilities(Collection accessExpressions = attributes.stream().map(attr -> AccessExpressionUtil.toAccessExpression(attr.getColumnVisibility())) .collect(Collectors.toSet()); return AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(MarkingFunctions.Factory.createMarkingFunctions(), - accessExpressions.stream().map(AccessExpressionUtil::fromAccessExpression).collect(Collectors.toList()))); + accessExpressions.stream().map(AccessExpressionUtil::toAccessExpressionMarkings).collect(Collectors.toList()))); } private long updateTimestamps() { diff --git a/warehouse/query-core/src/main/java/datawave/query/attributes/Document.java b/warehouse/query-core/src/main/java/datawave/query/attributes/Document.java index 5269e3c6330..ee954e63929 100644 --- a/warehouse/query-core/src/main/java/datawave/query/attributes/Document.java +++ b/warehouse/query-core/src/main/java/datawave/query/attributes/Document.java @@ -87,7 +87,7 @@ public MarkingFunctions getMarkingFunctions() { } public Markings getMarkings() { - return AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(getColumnVisibility())); + return AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(getColumnVisibility())); } public Document() { diff --git a/warehouse/query-core/src/main/java/datawave/query/common/grouping/GroupingUtils.java b/warehouse/query-core/src/main/java/datawave/query/common/grouping/GroupingUtils.java index 0f49c41aabc..fd0692a9772 100644 --- a/warehouse/query-core/src/main/java/datawave/query/common/grouping/GroupingUtils.java +++ b/warehouse/query-core/src/main/java/datawave/query/common/grouping/GroupingUtils.java @@ -48,7 +48,7 @@ public enum AverageAggregatorWriteFormat { public static ColumnVisibility combineVisibilities(Collection expressions, MarkingFunctions markingFunctions, boolean failOnError) { try { return AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(markingFunctions, - expressions.stream().map(AccessExpressionUtil::fromAccessExpression).collect(Collectors.toList()))); + expressions.stream().map(AccessExpressionUtil::toAccessExpressionMarkings).collect(Collectors.toList()))); } catch (MarkingFunctions.Exception e) { if (failOnError) { throw new IllegalArgumentException("Unable to combine access expressions: " + expressions, e); diff --git a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java index bfcca1f9840..7ab4df4477f 100644 --- a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java +++ b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java @@ -164,7 +164,7 @@ private DiscoveredThing aggregate(Collection termEntries) { // Otherwise, combine the visibilities, and return the aggregated result. try { ColumnVisibility visibility = AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(markingFunctions, - visibilities.stream().map(cv -> AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(cv))) + visibilities.stream().map(cv -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(cv))) .collect(Collectors.toSet()))); MapWritable countsByVis = new MapWritable(); visibilityToCounts.forEach((key, value) -> countsByVis.put(new Text(key), new LongWritable(value))); diff --git a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryTransformer.java index e37fbebe5ba..050521bece3 100644 --- a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryTransformer.java @@ -47,7 +47,7 @@ public EventBase transform(DiscoveredThing thing) { EventBase event = this.responseObjectFactory.getEvent(); Markings markings = AccessExpressionUtil - .fromAccessExpression(AccessExpressionUtil.toAccessExpression(new ColumnVisibility(thing.getColumnVisibility()))); + .toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(new ColumnVisibility(thing.getColumnVisibility()))); event.setMarkings(markings); List fields = new ArrayList<>(); @@ -64,7 +64,7 @@ public EventBase transform(DiscoveredThing thing) { if (thing.getCountsByColumnVisibility() != null && !thing.getCountsByColumnVisibility().isEmpty()) { for (Map.Entry entry : thing.getCountsByColumnVisibility().entrySet()) { Markings eMarkings = AccessExpressionUtil - .fromAccessExpression(AccessExpressionUtil.toAccessExpression(new ColumnVisibility(entry.getKey().toString()))); + .toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(new ColumnVisibility(entry.getKey().toString()))); fields.add(this.makeField("RECORD COUNT", eMarkings, entry.getKey().toString(), 0L, entry.getValue().toString())); } diff --git a/warehouse/query-core/src/main/java/datawave/query/discovery/TermInfoAggregation.java b/warehouse/query-core/src/main/java/datawave/query/discovery/TermInfoAggregation.java index 90cd4759527..cc9b8c97668 100644 --- a/warehouse/query-core/src/main/java/datawave/query/discovery/TermInfoAggregation.java +++ b/warehouse/query-core/src/main/java/datawave/query/discovery/TermInfoAggregation.java @@ -75,7 +75,7 @@ public DiscoveredThing apply(Collection from) { chosenCount = showReferenceCountInsteadOfTermCount ? referenceCount : termCount; try { - AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(ti.vis)); // just to test + AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(ti.vis)); // just to test // parsing columnVisibilities.add(ti.vis); @@ -112,7 +112,7 @@ public DiscoveredThing apply(Collection from) { columnVisibility = AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(MarkingFunctions.Factory.createMarkingFunctions(), columnVisibilities.stream() - .map(cv -> AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(cv))) + .map(cv -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(cv))) .collect(Collectors.toSet()))); } catch (Exception e) { diff --git a/warehouse/query-core/src/main/java/datawave/query/iterators/FieldIndexCountingIterator.java b/warehouse/query-core/src/main/java/datawave/query/iterators/FieldIndexCountingIterator.java index 5c364e6aac5..b3d640535a4 100644 --- a/warehouse/query-core/src/main/java/datawave/query/iterators/FieldIndexCountingIterator.java +++ b/warehouse/query-core/src/main/java/datawave/query/iterators/FieldIndexCountingIterator.java @@ -590,7 +590,7 @@ private Key buildReturnKey() { ColumnVisibility cv; try { cv = AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(MarkingFunctions.Factory.createMarkingFunctions(), - columnVisibilities.stream().map(c -> AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(c))) + columnVisibilities.stream().map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))) .collect(Collectors.toSet()))); } catch (MarkingFunctions.Exception e) { log.error("Could not combine visibilities: " + visibilitySet + " " + e); diff --git a/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java b/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java index 979e63569fb..d0f7bc50d33 100644 --- a/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java +++ b/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java @@ -472,7 +472,7 @@ private void addHitsForFunction(String field, ASTFunctionNode node, HitListArith try { cv = AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(MarkingFunctionsFactory.createMarkingFunctions(), cvs.stream() - .map(c -> AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(c))).collect(Collectors.toSet()))); + .map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))).collect(Collectors.toSet()))); } catch (MarkingFunctions.Exception e) { log.error("Failed to combine column visibilities while generating HIT_TERM for phrase function for field [" + field + "]"); log.error("msg: ", e); diff --git a/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java b/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java index 2e1384f92a6..3434dbc787d 100644 --- a/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java +++ b/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java @@ -294,7 +294,7 @@ public Attribute joinAttributes(String compositeName, Collection log.debug("dataList is " + dataList); } ColumnVisibility combinedColumnVisibility = AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(this.markingFunctions, - accessExpressions.stream().map(AccessExpressionUtil::fromAccessExpression).collect(Collectors.toList()))); + accessExpressions.stream().map(AccessExpressionUtil::toAccessExpressionMarkings).collect(Collectors.toList()))); metadata = new Key(metadata.getRow(), metadata.getColumnFamily(), new Text(), combinedColumnVisibility, timestamp); if (dataList.size() == 1) { return this.attrFactory.create(compositeName, dataList.get(0), metadata, toKeep, true); diff --git a/warehouse/query-core/src/main/java/datawave/query/table/parser/ContentKeyValueFactory.java b/warehouse/query-core/src/main/java/datawave/query/table/parser/ContentKeyValueFactory.java index d2de6672545..2f09dbc0072 100644 --- a/warehouse/query-core/src/main/java/datawave/query/table/parser/ContentKeyValueFactory.java +++ b/warehouse/query-core/src/main/java/datawave/query/table/parser/ContentKeyValueFactory.java @@ -7,11 +7,9 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.security.Authorizations; import org.apache.commons.io.IOUtils; import org.apache.log4j.Logger; -import datawave.marking.MarkingFunctions; import datawave.query.Constants; import datawave.query.table.parser.EventKeyValueFactory.EventKeyValue; @@ -19,7 +17,7 @@ public class ContentKeyValueFactory { private static final Logger log = Logger.getLogger(ContentKeyValueFactory.class); - public static ContentKeyValue parse(Key key, Value value, Authorizations auths, MarkingFunctions markingFunctions) throws MarkingFunctions.Exception { + public static ContentKeyValue parse(Key key, Value value) { if (null == key) throw new IllegalArgumentException("Cannot pass null key to ContentKeyValueFactory"); @@ -48,7 +46,7 @@ public static ContentKeyValue parse(Key key, Value value, Authorizations auths, c.setContents(contents); } - EventKeyValueFactory.parseColumnVisibility(c, key, auths, markingFunctions); + EventKeyValueFactory.parseColumnVisibility(c, key); return c; } diff --git a/warehouse/query-core/src/main/java/datawave/query/table/parser/EventKeyValueFactory.java b/warehouse/query-core/src/main/java/datawave/query/table/parser/EventKeyValueFactory.java index b9f18991897..1133e86a7ff 100644 --- a/warehouse/query-core/src/main/java/datawave/query/table/parser/EventKeyValueFactory.java +++ b/warehouse/query-core/src/main/java/datawave/query/table/parser/EventKeyValueFactory.java @@ -34,14 +34,13 @@ public static EventKeyValue parse(Key key, Value value, Authorizations auths, Ma e.setTimestamp(key.getTimestamp()); - parseColumnVisibility(e, key, auths, markingFunctions); + parseColumnVisibility(e, key); return e; } - protected static void parseColumnVisibility(EventKeyValue event, Key key, Authorizations auths, MarkingFunctions markingFunctions) - throws MarkingFunctions.Exception { - event.setMarkings(AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(new ColumnVisibility(key.getColumnVisibility())))); + protected static void parseColumnVisibility(EventKeyValue event, Key key) { + event.setMarkings(AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(new ColumnVisibility(key.getColumnVisibility())))); } public static class EventKeyValue { diff --git a/warehouse/query-core/src/main/java/datawave/query/table/parser/KeywordKeyValueFactory.java b/warehouse/query-core/src/main/java/datawave/query/table/parser/KeywordKeyValueFactory.java index 0ee5ff66b45..332242ae637 100644 --- a/warehouse/query-core/src/main/java/datawave/query/table/parser/KeywordKeyValueFactory.java +++ b/warehouse/query-core/src/main/java/datawave/query/table/parser/KeywordKeyValueFactory.java @@ -36,7 +36,7 @@ public static KeywordKeyValue parse(Key key, Value value, Authorizations auths, k.setContents(value.get()); } - EventKeyValueFactory.parseColumnVisibility(k, key, auths, markingFunctions); + EventKeyValueFactory.parseColumnVisibility(k, key); return k; } diff --git a/warehouse/query-core/src/main/java/datawave/query/table/parser/TermFrequencyKeyValueFactory.java b/warehouse/query-core/src/main/java/datawave/query/table/parser/TermFrequencyKeyValueFactory.java index 5a77edfed4a..895c2eebd98 100644 --- a/warehouse/query-core/src/main/java/datawave/query/table/parser/TermFrequencyKeyValueFactory.java +++ b/warehouse/query-core/src/main/java/datawave/query/table/parser/TermFrequencyKeyValueFactory.java @@ -56,9 +56,8 @@ public static TermFrequencyKeyValue parse(Key key, Value value, Authorizations a return t; } - protected static void parseColumnVisibility(TermFrequencyKeyValue tfkv, Key key, Authorizations auths, MarkingFunctions markingFunctions) - throws MarkingFunctions.Exception { - tfkv.setMarkings(AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(key.getColumnVisibilityParsed()))); + protected static void parseColumnVisibility(TermFrequencyKeyValue tfkv, Key key, Authorizations auths, MarkingFunctions markingFunctions) { + tfkv.setMarkings(AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(key.getColumnVisibilityParsed()))); } public static class TermFrequencyKeyValue extends EventKeyValue { diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountAggregatingIterator.java b/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountAggregatingIterator.java index 42f2753e8f6..6a7ab2bbe6e 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountAggregatingIterator.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountAggregatingIterator.java @@ -155,7 +155,7 @@ public Object getAggregatedEvent() { private ColumnVisibility getCombinedColumnVisibility() { try { return AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(markingFunctions, columnVisibilities.stream() - .map(c -> AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(c))).collect(Collectors.toSet()))); + .map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))).collect(Collectors.toSet()))); } catch (Exception e) { log.error("Could not combine columnVisibilities for the count", e); return null; diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountResultPostprocessor.java b/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountResultPostprocessor.java index af96643963e..a502128fd75 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountResultPostprocessor.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountResultPostprocessor.java @@ -74,7 +74,7 @@ public void apply(List results) { try { columnVisibility = AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(markingFunctions, columnVisibilities.stream() - .map(cv -> AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(cv))) + .map(cv -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(cv))) .collect(Collectors.toSet()))); } catch (Exception e) { log.error("Could not create combined columnVisibilities for the count", e); diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java index 50d65e29053..504b147dc63 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java @@ -510,7 +510,7 @@ public ColumnVisibility getColumnVisibility() { columnVisibilities.add(new ColumnVisibility(t)); } return AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(tupleMarkingFunctions, columnVisibilities.stream() - .map(c -> AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(c))).collect(Collectors.toList()))); + .map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))).collect(Collectors.toList()))); } catch (MarkingFunctions.Exception e) { logger.error("Could not create combined column visibility for the count", e); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/ContentQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/ContentQueryTransformer.java index 8b41d1e8f1f..3bd93957df9 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/ContentQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/ContentQueryTransformer.java @@ -13,7 +13,6 @@ import datawave.core.query.logic.BaseQueryLogicTransformer; import datawave.marking.MarkingFunctions; -import datawave.marking.MarkingFunctions.Exception; import datawave.microservice.query.Query; import datawave.query.table.parser.ContentKeyValueFactory; import datawave.query.table.parser.ContentKeyValueFactory.ContentKeyValue; @@ -122,11 +121,7 @@ public EventBase transform(Entry entry) { } ContentKeyValue ckv; - try { - ckv = ContentKeyValueFactory.parse(entry.getKey(), entry.getValue(), auths, markingFunctions); - } catch (Exception e1) { - throw new IllegalArgumentException("Unable to parse visibility", e1); - } + ckv = ContentKeyValueFactory.parse(entry.getKey(), entry.getValue()); EventBase e = responseObjectFactory.getEvent(); FieldBase field = responseObjectFactory.getField(); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java index 78a5563a7bb..6d2c7fdf3d3 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java @@ -163,10 +163,9 @@ private EventBase _transform(Entry documentEntry) throws EmptyObje return output; } - protected EventBase buildResponse(Document document, Key documentKey, ColumnVisibility eventCV, String colf, String row, MarkingFunctions mf) - throws MarkingFunctions.Exception { + protected EventBase buildResponse(Document document, Key documentKey, ColumnVisibility eventCV, String colf, String row, MarkingFunctions mf) { - Markings markings = AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(eventCV)); + Markings markings = AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(eventCV)); EventBase event = null; final Collection> documentFields = buildDocumentFields(documentKey, null, document, eventCV, mf); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformerSupport.java b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformerSupport.java index 6482e264f1b..8d2b9133cd4 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformerSupport.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformerSupport.java @@ -15,6 +15,7 @@ import java.util.Map.Entry; import java.util.Set; +import org.apache.accumulo.access.impl.AccessExpressionImpl; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.ColumnVisibility; @@ -71,7 +72,7 @@ public abstract class DocumentTransformerSupport extends EventQueryTransfor protected Boolean reducedResponse; private static final Logger log = Logger.getLogger(DocumentTransformerSupport.class); - private static final Markings EMPTY_MARKINGS = AccessExpressionMarkings.builder().build(); + private static final Markings EMPTY_MARKINGS = AccessExpressionMarkings.builder().accessExpression(AccessExpressionImpl.EMPTY).build(); private long sourceCount = 0; private long nextCount = 0; @@ -466,7 +467,7 @@ protected Collection> buildDocumentFields(Key documentKey, String f // Use the markings on the Field if we're returning the markings to the client if (!this.reducedResponse) { try { - Markings markings = AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(attr.getColumnVisibility())); + Markings markings = AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(attr.getColumnVisibility())); FieldBase field = this.makeField(fieldName, markings, attr.getColumnVisibility(), attr.getTimestamp(), attr.getData()); myFields.add(field); } catch (Exception ex) { diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformer.java index f399ae12f0e..7070534f10e 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformer.java @@ -42,7 +42,7 @@ public EdgeBase transform(Entry entry) { boolean statsEdge = edgeKey.isStatsKey(); try { Markings markings = AccessExpressionUtil - .fromAccessExpression(AccessExpressionUtil.toAccessExpression(new ColumnVisibility(edgeKey.getColvis()))); + .toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(new ColumnVisibility(edgeKey.getColvis()))); edge.setMarkings(markings); edge.setEdgeType(edgeKey.getType()); edge.setEdgeRelationship(edgeKey.getRelationship()); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java index c57721ea546..955cf9ca081 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java @@ -54,7 +54,7 @@ public EventBase transform(Entry entry) { EventBase event = this.responseObjectFactory.getEvent(); Markings markings = AccessExpressionUtil - .fromAccessExpression(AccessExpressionUtil.toAccessExpression(new ColumnVisibility(key.getColumnVisibility()))); + .toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(new ColumnVisibility(key.getColumnVisibility()))); if (null == markings || markings.isEmpty()) { // can't process this one because we did not have valid security markings log.error("Transformer visibility interpreter was null, skipping entry"); @@ -95,7 +95,7 @@ public EventBase transform(Entry entry) { } for (FieldValue fv : e.getValue()) { - Markings fieldMarkings = AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(fv.getVisibility())); + Markings fieldMarkings = AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(fv.getVisibility())); markingsToMerge.add(fieldMarkings); String value = new String(fv.getValue(), StandardCharsets.UTF_8); // if this is a content field name, then replace the value with the uid diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java index 02e770edce4..4c1c95875b9 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java @@ -128,7 +128,7 @@ protected Collection buildFacets(Key documentKey, String f FieldCardinalityBase fc = this.responseObjectFactory.getFieldCardinality(); fc.setField(v.getFieldName()); - fc.setMarkings(AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(attr.getColumnVisibility()))); + fc.setMarkings(AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(attr.getColumnVisibility()))); fc.setColumnVisibility(AccessExpressionUtil.normalize(AccessExpressionUtil.toAccessExpression(fc.getMarkings())).getExpression()); fc.setLower(v.getFloorValue()); fc.setUpper(v.getCeilingValue()); @@ -146,14 +146,13 @@ protected Collection buildFacets(Key documentKey, String f return myFields; } - protected FacetsBase buildResponse(Document document, Key documentKey, ColumnVisibility eventCV, String colf, String row, MarkingFunctions mf) - throws MarkingFunctions.Exception { + protected FacetsBase buildResponse(Document document, Key documentKey, ColumnVisibility eventCV, String colf, String row, MarkingFunctions mf) { FacetsBase facetedResponse = responseObjectFactory.getFacets(); final Collection documentFields = buildFacets(documentKey, null, document, eventCV, mf); - facetedResponse.setMarkings(AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(eventCV))); + facetedResponse.setMarkings(AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(eventCV))); facetedResponse.setFields(new ArrayList<>(documentFields)); // assign an estimate of the event size based on the document size @@ -179,7 +178,7 @@ public BaseQueryResponse createResponse(List resultList) { for (FieldCardinalityBase fcb : facet.getFields()) { if (StringUtils.isNotBlank(fcb.getColumnVisibility())) { combinedMarkings.add(AccessExpressionUtil - .fromAccessExpression(AccessExpressionUtil.toAccessExpression(new ColumnVisibility(fcb.getColumnVisibility())))); + .toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(new ColumnVisibility(fcb.getColumnVisibility())))); } } } diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/FieldIndexCountQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/FieldIndexCountQueryTransformer.java index e5a363d1069..64813fc7e8f 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/FieldIndexCountQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/FieldIndexCountQueryTransformer.java @@ -61,7 +61,7 @@ public EventBase transform(Entry entry) { String key = entry.getKey(); Tuple val = entry.getValue(); - Markings markings = AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(val.getColumnVisibility())); + Markings markings = AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(val.getColumnVisibility())); if (null == markings || markings.isEmpty()) { // can't process this one because we did not have valid security markings log.error("Transformer visibility interpreter was null, skipping entry"); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/GroupingDocumentTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/GroupingDocumentTransformer.java index 9105545637f..9aae433f36b 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/GroupingDocumentTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/GroupingDocumentTransformer.java @@ -132,7 +132,7 @@ private Multimap getFieldToFieldWithGroupingContextMap(Collection FieldBase created = null; try { created = this.makeField(shortName, - AccessExpressionUtil.fromAccessExpression( + AccessExpressionUtil.toAccessExpressionMarkings( AccessExpressionUtil.toAccessExpression(new ColumnVisibility(field.getColumnVisibility()))), field.getColumnVisibility(), 0L, field.getValueOfTypedValue()); } catch (Exception ex) { diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/ShardIndexQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/ShardIndexQueryTransformer.java index 71f7e755ea4..07d99020b04 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/ShardIndexQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/ShardIndexQueryTransformer.java @@ -67,7 +67,7 @@ public EventBase transform(Entry input) { EventBase event = responseObjectFactory.getEvent(); ColumnVisibility columnVisibility = new ColumnVisibility(entry.getKey().getColumnVisibility()); - Markings markings = AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(columnVisibility)); + Markings markings = AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(columnVisibility)); event.setMarkings(markings); List fields = new ArrayList<>(); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/ShardQueryCountTableTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/ShardQueryCountTableTransformer.java index 2e24ece9ea0..186d81d8f78 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/ShardQueryCountTableTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/ShardQueryCountTableTransformer.java @@ -47,7 +47,7 @@ public EventBase transform(Entry untypedEntry) { Long count = untypedEntry.getKey(); ColumnVisibility vis = untypedEntry.getValue(); - Markings markings = AccessExpressionUtil.fromAccessExpression(AccessExpressionUtil.toAccessExpression(vis)); + Markings markings = AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(vis)); EventBase e = this.responseObjectFactory.getEvent(); e.setMarkings(markings); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/TagCloudTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/TagCloudTransformer.java index ef0b1bfc1a7..72414f6115a 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/TagCloudTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/TagCloudTransformer.java @@ -204,7 +204,7 @@ private void configureV2TagCloud(TagCloudBase base, TagCloud tagCloud) { @SuppressWarnings("unchecked") private void convertAndSetMarkings(TagCloudBase base, AccessExpression visibility) { if (visibility != null && !visibility.getExpression().isEmpty()) { - base.setMarkings(AccessExpressionUtil.fromAccessExpression(visibility)); + base.setMarkings(AccessExpressionUtil.toAccessExpressionMarkings(visibility)); } } diff --git a/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleField.java b/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleField.java index 5994c3aadde..64cb4030dae 100644 --- a/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleField.java +++ b/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleField.java @@ -27,6 +27,8 @@ import io.protostuff.Output; import io.protostuff.Schema; +import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; + /** * This object is contained inside of Event objects to describe name/value pairs of data in the Event. */ @@ -227,7 +229,7 @@ public void mergeFrom(Input input, SimpleField message) throws IOException { case 5: String expr = input.readString(); if (expr != null && !expr.isEmpty()) { - message.markings = AccessExpressionMarkings.builder().accessExpression(MarkingFunctions.Default.ACCESS.newExpression(expr)).build(); + message.markings = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression(expr)).build(); } default: input.handleUnknownField(number, this); diff --git a/warehouse/query-core/src/test/java/datawave/query/common/grouping/DocumentGrouperTest.java b/warehouse/query-core/src/test/java/datawave/query/common/grouping/DocumentGrouperTest.java index 86d78e91b0e..e27697d3a7b 100644 --- a/warehouse/query-core/src/test/java/datawave/query/common/grouping/DocumentGrouperTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/common/grouping/DocumentGrouperTest.java @@ -11,6 +11,7 @@ import org.apache.accumulo.access.Access; import org.apache.accumulo.access.AccessExpression; +import org.apache.accumulo.access.examples.ParseExamples; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.security.ColumnVisibility; import org.assertj.core.util.Sets; @@ -40,7 +41,7 @@ public class DocumentGrouperTest { private static final ColumnVisibility COLVIS_I = new ColumnVisibility("I"); private static final ColumnVisibility COLVIS_ALL_E_I = new ColumnVisibility("ALL&E&I"); - private static final Access ACCESS = Access.builder().build(); + private static final Access ACCESS = ParseExamples.ACCESS; private static final AccessExpression AE_ALL = ACCESS.newExpression("ALL"); private static final AccessExpression AE_E = ACCESS.newExpression("E"); private static final AccessExpression AE_I = ACCESS.newExpression("I"); diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/keyword/TagCloudTestUtil.java b/warehouse/query-core/src/test/java/datawave/query/tables/keyword/TagCloudTestUtil.java index 919d77eceab..ccbd20fa6a4 100644 --- a/warehouse/query-core/src/test/java/datawave/query/tables/keyword/TagCloudTestUtil.java +++ b/warehouse/query-core/src/test/java/datawave/query/tables/keyword/TagCloudTestUtil.java @@ -11,12 +11,13 @@ import datawave.query.tables.keyword.transform.KeywordResultsTransformer; import datawave.webservice.result.keyword.DefaultTagCloud; import datawave.webservice.result.keyword.DefaultTagCloudEntry; +import org.apache.accumulo.access.examples.ParseExamples; /** * Utility for producing TagClouds and supporting objects for tests */ public class TagCloudTestUtil { - private static final Access ACCESS = Access.builder().build(); + private static final Access ACCESS = ParseExamples.ACCESS; private static final Markings ALL_MARKINGS = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression("ALL")).build(); public static final String SOPRANO_SOURCE = "20130101_0/test/-1kfeoq.-80b5fs.r0262j"; diff --git a/warehouse/query-core/src/test/java/datawave/query/transformer/FieldRenameTransformTest.java b/warehouse/query-core/src/test/java/datawave/query/transformer/FieldRenameTransformTest.java index d9e179e3153..98e1c356b46 100644 --- a/warehouse/query-core/src/test/java/datawave/query/transformer/FieldRenameTransformTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/transformer/FieldRenameTransformTest.java @@ -21,7 +21,7 @@ public class FieldRenameTransformTest { @Test - public void renameIdentityMapTest() throws MarkingFunctions.Exception { + public void renameIdentityMapTest() { Key key = new Key("shard", "dataType" + Constants.NULL + "uid"); Set fieldMap = new HashSet<>(); @@ -44,7 +44,7 @@ public void renameIdentityMapTest() throws MarkingFunctions.Exception { } @Test - public void renameFieldMapTest() throws MarkingFunctions.Exception { + public void renameFieldMapTest() { Key key = new Key("shard", "dataType" + Constants.NULL + "uid"); Set fieldMap = new HashSet<>(); @@ -70,7 +70,7 @@ public void renameFieldMapTest() throws MarkingFunctions.Exception { } @Test - public void renameFieldMapPreexistingTest() throws MarkingFunctions.Exception { + public void renameFieldMapPreexistingTest() { Key key = new Key("shard", "dataType" + Constants.NULL + "uid"); Set fieldMap = new HashSet<>(); @@ -96,7 +96,7 @@ public void renameFieldMapPreexistingTest() throws MarkingFunctions.Exception { } @Test - public void renameWithGroupingContextAndMultipleMappings() throws MarkingFunctions.Exception { + public void renameWithGroupingContextAndMultipleMappings() { Key key = new Key("shard", "dataType" + Constants.NULL + "uid"); Set fieldMap = new HashSet<>(); From 406a9143154b757103a1154882f13e3a32fd719f Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Thu, 26 Mar 2026 14:35:39 -0400 Subject: [PATCH 12/63] remove duplicate mergeMarking like methods --- .../common/util/AccessExpressionUtil.java | 32 ++++++++++++++++ .../data/config/ingest/CompositeIngest.java | 37 ++----------------- .../data/config/ingest/VirtualIngest.java | 37 ++----------------- .../handler/edge/define/EdgeDataBundle.java | 35 ++---------------- 4 files changed, 43 insertions(+), 98 deletions(-) diff --git a/core/common-util/src/main/java/datawave/core/common/util/AccessExpressionUtil.java b/core/common-util/src/main/java/datawave/core/common/util/AccessExpressionUtil.java index 277afbca541..f9021717fad 100644 --- a/core/common-util/src/main/java/datawave/core/common/util/AccessExpressionUtil.java +++ b/core/common-util/src/main/java/datawave/core/common/util/AccessExpressionUtil.java @@ -3,6 +3,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import java.util.Collection; +import java.util.List; import org.apache.accumulo.access.Access; import org.apache.accumulo.access.AccessExpression; @@ -160,4 +161,35 @@ public static AccessExpression emptyExpression() { public static Markings combine(MarkingFunctions markingFunctions, Collection> markings) throws MarkingFunctions.Exception { return ((MarkingFunctions) markingFunctions).combine(markings); } + + /** + * A helper routine to merge markings when merging fields of a NormalizedContentInterface + * + * @param markings1 + * markings + * @param markings2 + * different markings + * @return the merged markings + */ + public static Markings mergeMarkings(MarkingFunctions markingFunctions, Markings markings1, Markings markings2) { + if (markings2 != null) { + if (markings1 == null) { + markings1 = markings2; + } else { + try { + if (markings1 instanceof AccessExpressionMarkings && markings2 instanceof AccessExpressionMarkings) { + AccessExpressionMarkings aem1 = (AccessExpressionMarkings) markings1; + AccessExpressionMarkings aem2 = (AccessExpressionMarkings) markings2; + return combine(markingFunctions, List.of(aem1, aem2)); + } else { + throw new RuntimeException( + String.format("Unknown markings class %s or %s", markings1.getClass().getName(), markings1.getClass().getName())); + } + } catch (MarkingFunctions.Exception e) { + throw new RuntimeException("Unable to combine markings.", e); + } + } + } + return markings1; + } } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CompositeIngest.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CompositeIngest.java index f19a848bac8..2aaa168600a 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CompositeIngest.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CompositeIngest.java @@ -34,6 +34,8 @@ import datawave.marking.MarkingFunctions; import datawave.marking.Markings; +import static datawave.core.common.util.AccessExpressionUtil.mergeMarkings; + /** * Similar to VirtualIngest virtual fields, but composite fields are not written in the event section of the shard table (only in the field index section) * @@ -288,37 +290,6 @@ private String[] getConfPrefixes(String type) { return new String[] {str2, str1}; } - /** - * A helper routine to merge markings when merging fields of a NormalizedContentInterface - * - * @param markings1 - * markings - * @param markings2 - * different markings - * @return the merged markings - */ - protected Markings mergeMarkings(Markings markings1, Markings markings2) { - if (markings2 != null) { - if (markings1 == null) { - markings1 = markings2; - } else { - try { - if (markings1 instanceof AccessExpressionMarkings && markings2 instanceof AccessExpressionMarkings) { - AccessExpressionMarkings aem1 = (AccessExpressionMarkings) markings1; - AccessExpressionMarkings aem2 = (AccessExpressionMarkings) markings2; - return AccessExpressionUtil.combine(markingFunctions, List.of(aem1, aem2)); - } else { - throw new RuntimeException( - String.format("Unknown markings class %s or %s", markings1.getClass().getName(), markings1.getClass().getName())); - } - } catch (MarkingFunctions.Exception e) { - throw new RuntimeException("Unable to combine markings.", e); - } - } - } - return markings1; - } - public Multimap normalize(Multimap fields) { Multimap eventFields = HashMultimap.create(); for (Entry field : fields.entries()) { @@ -417,7 +388,7 @@ public void addCompositeFields(List compositeFields, if (pos + 1 < fields.length) { addCompositeFields(compositeFields, eventFields, compositeFieldName.replace("*", replacement), replacement, (grouping == null ? newGrouping : grouping), groupingPolicy, allowMissing, fields, pos + 1, originalValue, - normalizedValue, mergeMarkings(markings, value.getMarkings()), isOverloadedField); + normalizedValue, mergeMarkings(markingFunctions, markings, value.getMarkings()), isOverloadedField); } originalValue.setLength(oLen); normalizedValue.setLength(nLen); @@ -449,7 +420,7 @@ public void addCompositeFields(List compositeFields, } addCompositeFields(compositeFields, eventFields, compositeFieldName, replacement, (grouping == null ? newGrouping : grouping), groupingPolicy, allowMissing, fields, pos + 1, originalValue, normalizedValue, - mergeMarkings(markings, value.getMarkings()), isOverloadedField); + mergeMarkings(markingFunctions, markings, value.getMarkings()), isOverloadedField); originalValue.setLength(oLen); normalizedValue.setLength(nLen); } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java index 436d0c07464..88a1f5a4abc 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java @@ -32,6 +32,8 @@ import datawave.marking.MarkingFunctions; import datawave.marking.Markings; +import static datawave.core.common.util.AccessExpressionUtil.mergeMarkings; + public interface VirtualIngest { String VIRTUAL_FIELD_NAMES = VirtualFieldNormalizer.VIRTUAL_FIELD_NAMES; @@ -341,37 +343,6 @@ private String[] getConfPrefixes(String type, String instance) { } } - /** - * A helper routine to merge markings when merging fields of a NormalizedContentInterface - * - * @param markings1 - * markings - * @param markings2 - * different markings - * @return the merged markings - */ - protected Markings mergeMarkings(Markings markings1, Markings markings2) { - if (markings2 != null) { - if (markings1 == null) { - markings1 = markings2; - } else { - try { - if (markings1 instanceof AccessExpressionMarkings && markings2 instanceof AccessExpressionMarkings) { - AccessExpressionMarkings aem1 = (AccessExpressionMarkings) markings1; - AccessExpressionMarkings aem2 = (AccessExpressionMarkings) markings2; - return AccessExpressionUtil.combine(markingFunctions, List.of(aem1, aem2)); - } else { - throw new RuntimeException( - String.format("Unknown markings class %s or %s", markings1.getClass().getName(), markings1.getClass().getName())); - } - } catch (MarkingFunctions.Exception e) { - throw new RuntimeException("Unable to combine markings.", e); - } - } - } - return markings1; - } - /** * Create the normalized form of a map of fields, and add the virtual fields as configured above. * @@ -612,7 +583,7 @@ public void addVirtualFields(List virtualFields, Mul addVirtualFields(virtualFields, eventFields, groupings, virtualFieldName.replace("*", replacement), replacement, (grouping == null ? newGrouping : grouping), groupingPolicy, allowMissing, fields, pos + 1, this.defaultStartSeparator, this.defaultEndSeparator, originalValue, normalizedValue, - mergeMarkings(markings, value.getMarkings())); + mergeMarkings(markingFunctions, markings, value.getMarkings())); // reset the values to the original length originalValue.setLength(oLen); @@ -650,7 +621,7 @@ public void addVirtualFields(List virtualFields, Mul // recurse on the next virtual field segment addVirtualFields(virtualFields, eventFields, groupings, virtualFieldName, replacement, (grouping == null ? newGrouping : grouping), groupingPolicy, allowMissing, fields, pos + 1, this.defaultStartSeparator, this.defaultEndSeparator, originalValue, - normalizedValue, mergeMarkings(markings, value.getMarkings())); + normalizedValue, mergeMarkings(markingFunctions, markings, value.getMarkings())); // reset the values to the original length originalValue.setLength(oLen); diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/EdgeDataBundle.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/EdgeDataBundle.java index b00c4bd48b2..8be3c759a6c 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/EdgeDataBundle.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/EdgeDataBundle.java @@ -24,6 +24,8 @@ import datawave.marking.Markings; import datawave.util.time.DateHelper; +import static datawave.core.common.util.AccessExpressionUtil.mergeMarkings; + /** * Combines an EdgeDefinition with values obtained from Event data. * @@ -96,7 +98,7 @@ public EdgeDataBundle(EdgeDefinition edgeDef, NormalizedContentInterface ifaceSo // even though event, etc. references are saved above, passing in the event // prevents future bug this.initFieldMasking(helper, event); - this.initMarkings(getSource().getMarkings(), getSink().getMarkings()); + this.markings = mergeMarkings(markingFunctions, getSource().getMarkings(), getSink().getMarkings()); } private int getHour(long time) { @@ -141,37 +143,6 @@ public void setMaskedVisibility(ColumnVisibility maskedVisibility) { this.maskedVisibility = maskedVisibility; } - /** - * A helper routine to merge markings when merging fields of a NormalizedContentInterface - * - * @param markings1 - * markings - * @param markings2 - * different markings - */ - protected void initMarkings(Markings markings1, Markings markings2) { - if (markings2 != null) { - if (markings1 == null) { - this.markings = markings2; - } else { - try { - if (markings1 instanceof AccessExpressionMarkings && markings2 instanceof AccessExpressionMarkings) { - AccessExpressionMarkings aem1 = (AccessExpressionMarkings) markings1; - AccessExpressionMarkings aem2 = (AccessExpressionMarkings) markings2; - this.markings = AccessExpressionUtil.combine(markingFunctions, List.of(aem1, aem2)); - } else { - throw new RuntimeException( - String.format("Unknown markings class %s or %s", markings1.getClass().getName(), markings1.getClass().getName())); - } - } catch (MarkingFunctions.Exception e) { - throw new RuntimeException("Unable to combine markings.", e); - } - } - } else { - this.markings = markings1; - } - } - public int getDuration() { return null == this.getDurationValue() ? -1 : this.getDurationValue().getDuration(); } From ad2ef6cdf36f82f5173744e080450ff03cdb0f9c Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Fri, 27 Mar 2026 13:23:36 -0400 Subject: [PATCH 13/63] rename mf to markingFunctions for consistency --- .../query/tables/shard/FieldIndexCountQueryLogic.java | 4 ++-- .../java/datawave/query/transformer/DocumentTransformer.java | 4 ++-- .../java/datawave/query/transformer/FacetedTransformer.java | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java index 504b147dc63..caf38688bad 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java @@ -483,8 +483,8 @@ public static class Tuple { private long maxTimestamp = 0L; Set uniqueVisibilities = new HashSet<>(); - public Tuple(MarkingFunctions mf) { - tupleMarkingFunctions = mf; + public Tuple(MarkingFunctions markingFunctions) { + tupleMarkingFunctions = markingFunctions; } public void aggregate(Key key, Value val) { diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java index 6d2c7fdf3d3..667c22068d6 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java @@ -163,12 +163,12 @@ private EventBase _transform(Entry documentEntry) throws EmptyObje return output; } - protected EventBase buildResponse(Document document, Key documentKey, ColumnVisibility eventCV, String colf, String row, MarkingFunctions mf) { + protected EventBase buildResponse(Document document, Key documentKey, ColumnVisibility eventCV, String colf, String row, MarkingFunctions markingFunctions) { Markings markings = AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(eventCV)); EventBase event = null; - final Collection> documentFields = buildDocumentFields(documentKey, null, document, eventCV, mf); + final Collection> documentFields = buildDocumentFields(documentKey, null, document, eventCV, markingFunctions); // if documentFields is empty, then the response contained only timing metadata if (!documentFields.isEmpty()) { event = this.responseObjectFactory.getEvent(); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java index 4c1c95875b9..698eae50691 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java @@ -146,11 +146,11 @@ protected Collection buildFacets(Key documentKey, String f return myFields; } - protected FacetsBase buildResponse(Document document, Key documentKey, ColumnVisibility eventCV, String colf, String row, MarkingFunctions mf) { + protected FacetsBase buildResponse(Document document, Key documentKey, ColumnVisibility eventCV, String colf, String row, MarkingFunctions markingFunctions) { FacetsBase facetedResponse = responseObjectFactory.getFacets(); - final Collection documentFields = buildFacets(documentKey, null, document, eventCV, mf); + final Collection documentFields = buildFacets(documentKey, null, document, eventCV, markingFunctions); facetedResponse.setMarkings(AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(eventCV))); facetedResponse.setFields(new ArrayList<>(documentFields)); From a1f6fc7c9695c03877d86253f43afa0948136203 Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Mon, 30 Mar 2026 10:04:16 -0400 Subject: [PATCH 14/63] formatting --- .../marking/ColumnVisibilitySecurityMarking.java | 4 ++-- .../query/cachedresults/CacheableQueryRowReader.java | 7 +++---- .../datawave/modification/MutableMetadataHandler.java | 4 ++-- .../java/datawave/marking/AccessExpressionMarkings.java | 2 +- .../datawave/iterators/FrequencyMetadataAggregator.java | 5 +++-- .../datawave/webservice/response/objects/DefaultKey.java | 2 +- .../datawave/webservice/response/LookupResponseTest.java | 4 +--- .../microservice/accumulo/admin/AdminService.java | 1 + .../microservice/accumulo/lookup/LookupService.java | 4 ++-- .../webservice/dictionary/data/DefaultDescription.java | 5 ++--- .../webservice/metadata/DefaultMetadataField.java | 1 - .../dictionary/DataDictionaryControllerLogic.java | 5 ++--- .../metadata/DefaultMetadataFieldScanner.java | 7 +++---- .../metadata/MetadataDescriptionsHelper.java | 9 +++------ .../microservice/querymetric/QueryMetricTest.java | 1 - .../handler/ShardTableQueryMetricHandler.java | 1 - .../microservice/querymetric/MetadataCachingTest.java | 1 - .../microservice/querymetric/QueryMetricTestBase.java | 3 --- .../microservice/query/AbstractQueryServiceTest.java | 2 -- .../microservice/query/lookup/LookupServiceTest.java | 1 - .../ingest/data/config/ingest/CompositeIngest.java | 6 ++---- .../ingest/data/config/ingest/VirtualIngest.java | 6 ++---- .../mapreduce/handler/edge/define/EdgeDataBundle.java | 6 ++---- .../datawave/ingest/csv/mr/input/CSVRecordReader.java | 5 ++--- .../java/datawave/util/keyword/DefaultTagCloudUtils.java | 4 ++-- .../src/main/java/datawave/util/keyword/TagCloud.java | 4 ++-- .../FieldIndexCountingIteratorPerVisibility.java | 3 ++- .../core/iterators/GlobalIndexDateSummaryIterator.java | 3 ++- .../java/datawave/query/discovery/DiscoveryIterator.java | 3 ++- .../datawave/query/discovery/TermInfoAggregation.java | 2 +- .../java/datawave/query/jexl/DatawaveInterpreter.java | 5 +++-- .../query/table/parser/EventKeyValueFactory.java | 3 ++- .../query/tables/shard/CountAggregatingIterator.java | 6 ++++-- .../query/tables/shard/FieldIndexCountQueryLogic.java | 6 ++++-- .../datawave/query/transformer/DocumentTransformer.java | 3 ++- .../datawave/query/transformer/FacetedTransformer.java | 3 ++- .../webservice/query/result/event/SimpleField.java | 5 ++--- .../datawave/query/tables/keyword/TagCloudTestUtil.java | 2 +- .../query/transformer/FieldRenameTransformTest.java | 1 - 39 files changed, 65 insertions(+), 80 deletions(-) diff --git a/core/base-rest-responses/src/main/java/datawave/marking/ColumnVisibilitySecurityMarking.java b/core/base-rest-responses/src/main/java/datawave/marking/ColumnVisibilitySecurityMarking.java index c3639140978..8c1a0eab638 100644 --- a/core/base-rest-responses/src/main/java/datawave/marking/ColumnVisibilitySecurityMarking.java +++ b/core/base-rest-responses/src/main/java/datawave/marking/ColumnVisibilitySecurityMarking.java @@ -11,11 +11,11 @@ import org.apache.accumulo.access.Access; import org.apache.accumulo.access.AccessExpression; - -import com.google.common.base.Preconditions; import org.apache.accumulo.access.examples.ParseExamples; import org.apache.accumulo.access.impl.AccessExpressionImpl; +import com.google.common.base.Preconditions; + @XmlAccessorType(XmlAccessType.NONE) @XmlAccessorOrder(XmlAccessOrder.ALPHABETICAL) public class ColumnVisibilitySecurityMarking implements SecurityMarking { diff --git a/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowReader.java b/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowReader.java index 0f9fd6e0646..b60c8bcf6f3 100644 --- a/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowReader.java +++ b/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowReader.java @@ -1,5 +1,7 @@ package datawave.core.query.cachedresults; +import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; + import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; @@ -18,8 +20,6 @@ import datawave.webservice.query.cachedresults.CacheableQueryRow; import datawave.webservice.query.result.event.ResponseObjectFactory; -import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; - public class CacheableQueryRowReader { private static Logger log = LoggerFactory.getLogger(CacheableQueryRowReader.class); @@ -109,8 +109,7 @@ public static CacheableQueryRow createRow(ResultSet cachedRowSet, Set fi columnMarkingsMap.put(columnName, AccessExpressionUtil.toAccessExpressionMarkings(ACCESS.newExpression(markingStr))); columnVisibilityMap.put(columnName, combinedString.substring(x + 1)); } else { - columnMarkingsMap.put(columnName, - AccessExpressionUtil.toAccessExpressionMarkings(ACCESS.newExpression(combinedString))); + columnMarkingsMap.put(columnName, AccessExpressionUtil.toAccessExpressionMarkings(ACCESS.newExpression(combinedString))); columnVisibilityMap.put(columnName, ""); } } diff --git a/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java b/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java index cb8c8bfa8ea..a89cb82151b 100644 --- a/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java +++ b/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java @@ -1,5 +1,7 @@ package datawave.modification; +import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; + import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -76,8 +78,6 @@ import datawave.webservice.result.EventQueryResponseBase; import datawave.webservice.result.GenericResponse; -import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; - /** * Class that handles requests for modification requests (INSERT, UPDATE, DELETE) for metadata in the shard schema.
*
diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java index 9458a9b6599..63ae8da07dc 100644 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java @@ -2,6 +2,7 @@ import org.apache.accumulo.access.Access; import org.apache.accumulo.access.AccessExpression; +import org.apache.accumulo.access.examples.ParseExamples; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonCreator; @@ -9,7 +10,6 @@ import lombok.Builder; import lombok.Data; -import org.apache.accumulo.access.examples.ParseExamples; @Data @Builder diff --git a/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java b/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java index 494d1646e66..256f12182ba 100644 --- a/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java +++ b/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java @@ -586,8 +586,9 @@ private Map buildCacheEntries() { private ColumnVisibility combineAllVisibilities() { Set visibilities = visibilityToDateFrequencies.keySet(); try { - return AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(markingFunctions, visibilities.stream() - .map(cv -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(cv))).collect(Collectors.toSet()))); + return AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(markingFunctions, + visibilities.stream().map(cv -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(cv))) + .collect(Collectors.toSet()))); } catch (MarkingFunctions.Exception e) { log.error("Failed to combine visibilities " + visibilities); throw new IllegalArgumentException("Failed to combine visibilities " + visibilities, e); diff --git a/microservices/services/accumulo/api/src/main/java/datawave/webservice/response/objects/DefaultKey.java b/microservices/services/accumulo/api/src/main/java/datawave/webservice/response/objects/DefaultKey.java index a4b00e69223..b720dba89f7 100644 --- a/microservices/services/accumulo/api/src/main/java/datawave/webservice/response/objects/DefaultKey.java +++ b/microservices/services/accumulo/api/src/main/java/datawave/webservice/response/objects/DefaultKey.java @@ -1,6 +1,7 @@ package datawave.webservice.response.objects; import static java.nio.charset.StandardCharsets.UTF_8; + import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; import javax.xml.bind.annotation.XmlAccessType; @@ -10,7 +11,6 @@ import org.apache.commons.codec.binary.Base64; import datawave.marking.AccessExpressionMarkings; -import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.webservice.query.util.TypedValue; diff --git a/microservices/services/accumulo/api/src/test/java/datawave/webservice/response/LookupResponseTest.java b/microservices/services/accumulo/api/src/test/java/datawave/webservice/response/LookupResponseTest.java index 75bcf85d522..3c0d318fb32 100644 --- a/microservices/services/accumulo/api/src/test/java/datawave/webservice/response/LookupResponseTest.java +++ b/microservices/services/accumulo/api/src/test/java/datawave/webservice/response/LookupResponseTest.java @@ -19,7 +19,6 @@ import com.fasterxml.jackson.module.jaxb.JaxbAnnotationIntrospector; import datawave.marking.AccessExpressionMarkings; -import datawave.marking.MarkingFunctions; import datawave.webservice.query.exception.QueryExceptionType; import datawave.webservice.response.objects.DefaultKey; import datawave.webservice.response.objects.Entry; @@ -40,8 +39,7 @@ private void responseSetup() { lookupResponse = new LookupResponse(); List entryList = new ArrayList<>(); for (int i = 0; i < 3; i++) { - AccessExpressionMarkings markings = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression("A&B&C" + i)) - .build(); + AccessExpressionMarkings markings = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression("A&B&C" + i)).build(); KeyBase responseKey = new DefaultKey(); responseKey.setRow("row" + i); responseKey.setColFam("cf" + i); diff --git a/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/admin/AdminService.java b/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/admin/AdminService.java index d1ad8f8b6e9..f2ec378d63d 100644 --- a/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/admin/AdminService.java +++ b/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/admin/AdminService.java @@ -1,6 +1,7 @@ package datawave.microservice.accumulo.admin; import static java.nio.charset.StandardCharsets.UTF_8; + import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; import java.util.ArrayList; diff --git a/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/lookup/LookupService.java b/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/lookup/LookupService.java index 67f73e63460..ded3233dc5b 100644 --- a/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/lookup/LookupService.java +++ b/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/lookup/LookupService.java @@ -1,6 +1,7 @@ package datawave.microservice.accumulo.lookup; import static java.nio.charset.StandardCharsets.UTF_8; + import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; import java.util.ArrayList; @@ -220,8 +221,7 @@ public LookupResponse lookup(LookupRequest request, DatawaveUserDetails currentU continue; } - final Markings markings = AccessExpressionUtil - .toAccessExpressionMarkings(ACCESS.newExpression(k.getColumnVisibility().toString())); + final Markings markings = AccessExpressionUtil.toAccessExpressionMarkings(ACCESS.newExpression(k.getColumnVisibility().toString())); final KeyBase responseKey = responseObjectFactory.createKey(); responseKey.setRow(currRow); diff --git a/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java b/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java index e187caeab26..dbc3f7fd792 100644 --- a/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java +++ b/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java @@ -1,5 +1,7 @@ package datawave.webservice.dictionary.data; +import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; + import java.io.IOException; import java.io.Serializable; import java.util.HashMap; @@ -15,7 +17,6 @@ import org.apache.accumulo.access.AccessExpression; import datawave.marking.AccessExpressionMarkings; -import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.webservice.query.result.event.MapSchema; import io.protostuff.Input; @@ -23,8 +24,6 @@ import io.protostuff.Output; import io.protostuff.Schema; -import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; - @XmlAccessorType(XmlAccessType.NONE) @XmlAccessorOrder(XmlAccessOrder.ALPHABETICAL) public class DefaultDescription extends DescriptionBase implements Serializable, Message { diff --git a/microservices/services/dictionary/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java b/microservices/services/dictionary/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java index 5a9f51b550a..b9221f9b8da 100644 --- a/microservices/services/dictionary/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java +++ b/microservices/services/dictionary/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java @@ -7,7 +7,6 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; import javax.xml.bind.annotation.XmlAccessType; diff --git a/microservices/services/dictionary/service/src/main/java/datawave/microservice/dictionary/DataDictionaryControllerLogic.java b/microservices/services/dictionary/service/src/main/java/datawave/microservice/dictionary/DataDictionaryControllerLogic.java index 47bb8d13484..1ce6203253f 100644 --- a/microservices/services/dictionary/service/src/main/java/datawave/microservice/dictionary/DataDictionaryControllerLogic.java +++ b/microservices/services/dictionary/service/src/main/java/datawave/microservice/dictionary/DataDictionaryControllerLogic.java @@ -1,5 +1,7 @@ package datawave.microservice.dictionary; +import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; + import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -15,7 +17,6 @@ import com.google.common.collect.Multimap; import datawave.marking.AccessExpressionMarkings; -import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.microservice.AccumuloConnectionService; import datawave.microservice.Connection; @@ -31,8 +32,6 @@ import datawave.webservice.metadata.MetadataFieldBase; import datawave.webservice.result.VoidResponse; -import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; - public class DataDictionaryControllerLogic,DICT extends DataDictionaryBase,META extends MetadataFieldBase,FIELD extends DictionaryFieldBase,FIELDS extends FieldsBase> { private final DataDictionaryProperties dataDictionaryConfiguration; private final DataDictionary dataDictionary; diff --git a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java index 2932f2140f2..f0f51742da7 100644 --- a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java +++ b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java @@ -1,5 +1,7 @@ package datawave.microservice.metadata; +import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; + import java.time.Instant; import java.time.ZoneId; import java.time.format.DateTimeFormatter; @@ -34,8 +36,6 @@ import datawave.webservice.dictionary.data.DefaultFields; import datawave.webservice.metadata.DefaultMetadataField; -import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; - public class DefaultMetadataFieldScanner { private static final Logger log = LoggerFactory.getLogger(DefaultMetadataFieldScanner.class); @@ -278,8 +278,7 @@ private void setFieldNameAndAlias() { private void setDescriptions() { DefaultDescription description = responseObjectFactory.getDescription(); description.setDescription(currValue.toString()); - description.setMarkings( - AccessExpressionUtil.toAccessExpressionMarkings(ACCESS.newExpression(currKey.getColumnVisibility().toString()))); + description.setMarkings(AccessExpressionUtil.toAccessExpressionMarkings(ACCESS.newExpression(currKey.getColumnVisibility().toString()))); currField.getDescriptions().add(description); } diff --git a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java index 80e2cc63682..de10da1ede8 100644 --- a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java +++ b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java @@ -85,8 +85,7 @@ public SetMultimap getDescriptions(Set ingestTypeFil return descs; } - public SetMultimap getFieldDescriptions(Set ingestTypeFilter) - throws TableNotFoundException, MarkingFunctions.Exception { + public SetMultimap getFieldDescriptions(Set ingestTypeFilter) throws TableNotFoundException, MarkingFunctions.Exception { SetMultimap descriptions = getDescriptions(ingestTypeFilter); SetMultimap fieldDescriptions = HashMultimap.create(); @@ -118,8 +117,7 @@ public void setDescription(MetadataEntry entry, DescriptionBase desc) setDescriptions(entry, Collections.singleton(desc)); } - public void setDescriptions(MetadataEntry entry, Set descs) - throws TableNotFoundException, MutationsRejectedException { + public void setDescriptions(MetadataEntry entry, Set descs) throws TableNotFoundException, MutationsRejectedException { BatchWriter bw = null; try { BatchWriterConfig bwConfig = new BatchWriterConfig().setMaxMemory(10000L).setMaxLatency(100L, TimeUnit.MILLISECONDS).setMaxWriteThreads(1); @@ -153,8 +151,7 @@ public void setDescriptions(MetadataEntry entry, Set * @throws MutationsRejectedException * if writing the update to Accumulo fails */ - public void removeDescription(MetadataEntry entry, DescriptionBase desc) - throws TableNotFoundException, MutationsRejectedException { + public void removeDescription(MetadataEntry entry, DescriptionBase desc) throws TableNotFoundException, MutationsRejectedException { BatchWriter bw = null; try { BatchWriterConfig bwConfig = new BatchWriterConfig().setMaxMemory(10000L).setMaxLatency(100L, TimeUnit.MILLISECONDS).setMaxWriteThreads(1); diff --git a/microservices/services/query-metric/api/src/test/java/datawave/microservice/querymetric/QueryMetricTest.java b/microservices/services/query-metric/api/src/test/java/datawave/microservice/querymetric/QueryMetricTest.java index 8b855965507..e100f3741b8 100644 --- a/microservices/services/query-metric/api/src/test/java/datawave/microservice/querymetric/QueryMetricTest.java +++ b/microservices/services/query-metric/api/src/test/java/datawave/microservice/querymetric/QueryMetricTest.java @@ -28,7 +28,6 @@ import com.fasterxml.jackson.module.jaxb.JaxbAnnotationModule; import datawave.marking.AccessExpressionMarkings; -import datawave.marking.MarkingFunctions; import datawave.microservice.querymetric.BaseQueryMetric.Lifecycle; import datawave.microservice.querymetric.BaseQueryMetric.PageMetric; import datawave.microservice.querymetric.BaseQueryMetric.Prediction; diff --git a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java index 8ee1f0d176f..5041a653288 100644 --- a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java +++ b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java @@ -19,7 +19,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantReadWriteLock; -import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; diff --git a/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/MetadataCachingTest.java b/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/MetadataCachingTest.java index 2d346df6339..d4b12b59e85 100644 --- a/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/MetadataCachingTest.java +++ b/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/MetadataCachingTest.java @@ -11,7 +11,6 @@ import java.util.stream.Collectors; import org.apache.accumulo.core.security.Authorizations; -import org.junit.Assert; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; diff --git a/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java b/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java index c8bd2741f63..2af6b5055a3 100644 --- a/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java +++ b/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java @@ -13,7 +13,6 @@ import java.util.Collection; import java.util.Collections; import java.util.Date; -import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -68,8 +67,6 @@ import datawave.core.common.connection.AccumuloConnectionFactory; import datawave.ingest.protobuf.Uid; import datawave.marking.AccessExpressionMarkings; -import datawave.marking.ColumnVisibilitySecurityMarking; -import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.microservice.authorization.preauth.ProxiedEntityX509Filter; import datawave.microservice.authorization.user.DatawaveUserDetails; diff --git a/microservices/services/query/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java b/microservices/services/query/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java index 1ddb7ad41bd..4d40c982f9d 100644 --- a/microservices/services/query/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java +++ b/microservices/services/query/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java @@ -17,7 +17,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -71,7 +70,6 @@ import com.hazelcast.core.HazelcastInstance; import datawave.marking.ColumnVisibilitySecurityMarking; -import datawave.marking.MarkingFunctions; import datawave.microservice.audit.AuditClient; import datawave.microservice.authorization.jwt.JWTRestTemplate; import datawave.microservice.authorization.user.DatawaveUserDetails; diff --git a/microservices/services/query/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java b/microservices/services/query/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java index 124a3d14398..86ec583f7b9 100644 --- a/microservices/services/query/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java +++ b/microservices/services/query/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java @@ -7,7 +7,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CompositeIngest.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CompositeIngest.java index 2aaa168600a..4dc8702a0cd 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CompositeIngest.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CompositeIngest.java @@ -1,5 +1,7 @@ package datawave.ingest.data.config.ingest; +import static datawave.core.common.util.AccessExpressionUtil.mergeMarkings; + import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -25,17 +27,13 @@ import com.google.common.collect.Multimap; import com.google.common.collect.Sets; -import datawave.core.common.util.AccessExpressionUtil; import datawave.ingest.data.Type; import datawave.ingest.data.config.GroupedNormalizedContentInterface; import datawave.ingest.data.config.NormalizedContentInterface; import datawave.ingest.data.config.NormalizedFieldAndValue; -import datawave.marking.AccessExpressionMarkings; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; -import static datawave.core.common.util.AccessExpressionUtil.mergeMarkings; - /** * Similar to VirtualIngest virtual fields, but composite fields are not written in the event section of the shard table (only in the field index section) * diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java index 88a1f5a4abc..03dd69cee04 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java @@ -1,5 +1,7 @@ package datawave.ingest.data.config.ingest; +import static datawave.core.common.util.AccessExpressionUtil.mergeMarkings; + import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -22,18 +24,14 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Multimap; -import datawave.core.common.util.AccessExpressionUtil; import datawave.data.type.NoOpType; import datawave.ingest.data.Type; import datawave.ingest.data.config.GroupedNormalizedContentInterface; import datawave.ingest.data.config.NormalizedContentInterface; import datawave.ingest.data.config.NormalizedFieldAndValue; -import datawave.marking.AccessExpressionMarkings; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; -import static datawave.core.common.util.AccessExpressionUtil.mergeMarkings; - public interface VirtualIngest { String VIRTUAL_FIELD_NAMES = VirtualFieldNormalizer.VIRTUAL_FIELD_NAMES; diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/EdgeDataBundle.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/EdgeDataBundle.java index 8be3c759a6c..df5c9512caa 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/EdgeDataBundle.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/EdgeDataBundle.java @@ -1,5 +1,7 @@ package datawave.ingest.mapreduce.handler.edge.define; +import static datawave.core.common.util.AccessExpressionUtil.mergeMarkings; + import java.io.IOException; import java.util.Calendar; import java.util.List; @@ -9,7 +11,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import datawave.core.common.util.AccessExpressionUtil; import datawave.edge.util.EdgeKey; import datawave.edge.util.EdgeValue.EdgeValueBuilder; import datawave.edge.util.EdgeValueHelper; @@ -19,13 +20,10 @@ import datawave.ingest.data.config.NormalizedContentInterface; import datawave.ingest.data.config.ingest.IngestHelperInterface; import datawave.ingest.mapreduce.handler.edge.define.VertexValue.ValueType; -import datawave.marking.AccessExpressionMarkings; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.util.time.DateHelper; -import static datawave.core.common.util.AccessExpressionUtil.mergeMarkings; - /** * Combines an EdgeDefinition with values obtained from Event data. * diff --git a/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java b/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java index 50c0f8c0f8d..3e6bb55905b 100644 --- a/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java +++ b/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java @@ -1,5 +1,7 @@ package datawave.ingest.csv.mr.input; +import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; + import java.io.IOException; import java.util.Map; import java.util.Map.Entry; @@ -29,9 +31,6 @@ import datawave.ingest.metadata.id.MetadataIdParser; import datawave.ingest.validation.EventValidator; import datawave.marking.AccessExpressionMarkings; -import datawave.marking.MarkingFunctions; - -import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; public class CSVRecordReader extends CSVReaderBase implements EventFixer { diff --git a/warehouse/keyword-common/src/main/java/datawave/util/keyword/DefaultTagCloudUtils.java b/warehouse/keyword-common/src/main/java/datawave/util/keyword/DefaultTagCloudUtils.java index b3e8388a5bf..233f38fa68d 100644 --- a/warehouse/keyword-common/src/main/java/datawave/util/keyword/DefaultTagCloudUtils.java +++ b/warehouse/keyword-common/src/main/java/datawave/util/keyword/DefaultTagCloudUtils.java @@ -1,5 +1,7 @@ package datawave.util.keyword; +import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; + import java.io.Serializable; import java.util.Collection; import java.util.Comparator; @@ -16,8 +18,6 @@ import datawave.marking.MarkingFunctions; import datawave.marking.Markings; -import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; - /** * Default implementations for pluggable utilities for generating tag clouds, includes mechanisms to partition keywords into separate tag clouds, combine or * merge visibility strings, and calculate scores, source collections and frequencies of individual keywords based on observed results diff --git a/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloud.java b/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloud.java index 5bcd58871ad..cd36a8c543c 100644 --- a/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloud.java +++ b/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloud.java @@ -1,5 +1,7 @@ package datawave.util.keyword; +import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; + import java.util.ArrayList; import java.util.Collection; import java.util.Comparator; @@ -16,8 +18,6 @@ import com.google.gson.Gson; -import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; - /** A tag cloud - a collection of tags that have a keyword, score, frequency and list of sources from which they originated */ public class TagCloud { diff --git a/warehouse/query-core/src/main/java/datawave/core/iterators/FieldIndexCountingIteratorPerVisibility.java b/warehouse/query-core/src/main/java/datawave/core/iterators/FieldIndexCountingIteratorPerVisibility.java index 4369144d98c..22cbd2f5f58 100644 --- a/warehouse/query-core/src/main/java/datawave/core/iterators/FieldIndexCountingIteratorPerVisibility.java +++ b/warehouse/query-core/src/main/java/datawave/core/iterators/FieldIndexCountingIteratorPerVisibility.java @@ -568,7 +568,8 @@ private Map buildReturnKeys() { try { // Calculate the columnVisibility for this key from the combiner. cv = AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(MarkingFunctions.Factory.createMarkingFunctions(), - columnVisibilities.stream().map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))) + columnVisibilities.stream() + .map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))) .collect(Collectors.toSet()))); } catch (Exception e) { log.error("Could not create combined columnVisibility for the count", e); diff --git a/warehouse/query-core/src/main/java/datawave/core/iterators/GlobalIndexDateSummaryIterator.java b/warehouse/query-core/src/main/java/datawave/core/iterators/GlobalIndexDateSummaryIterator.java index f5615e03bb9..898293dcc71 100644 --- a/warehouse/query-core/src/main/java/datawave/core/iterators/GlobalIndexDateSummaryIterator.java +++ b/warehouse/query-core/src/main/java/datawave/core/iterators/GlobalIndexDateSummaryIterator.java @@ -260,7 +260,8 @@ public Map getKeyValues() throws IOException { // Note that the access controls found in the combined ColumnVisibility will be pulled out appropriately here ColumnVisibility cv = AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine( MarkingFunctions.Factory.createMarkingFunctions(), - columnVisibilities.stream().map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))) + columnVisibilities.stream() + .map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))) .collect(Collectors.toSet()))); // Create a new Key compatible with the shardIndex key format diff --git a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java index 7ab4df4477f..934c82e8275 100644 --- a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java +++ b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java @@ -164,7 +164,8 @@ private DiscoveredThing aggregate(Collection termEntries) { // Otherwise, combine the visibilities, and return the aggregated result. try { ColumnVisibility visibility = AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(markingFunctions, - visibilities.stream().map(cv -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(cv))) + visibilities.stream() + .map(cv -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(cv))) .collect(Collectors.toSet()))); MapWritable countsByVis = new MapWritable(); visibilityToCounts.forEach((key, value) -> countsByVis.put(new Text(key), new LongWritable(value))); diff --git a/warehouse/query-core/src/main/java/datawave/query/discovery/TermInfoAggregation.java b/warehouse/query-core/src/main/java/datawave/query/discovery/TermInfoAggregation.java index cc9b8c97668..ae261366891 100644 --- a/warehouse/query-core/src/main/java/datawave/query/discovery/TermInfoAggregation.java +++ b/warehouse/query-core/src/main/java/datawave/query/discovery/TermInfoAggregation.java @@ -76,7 +76,7 @@ public DiscoveredThing apply(Collection from) { try { AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(ti.vis)); // just to test - // parsing + // parsing columnVisibilities.add(ti.vis); // Keep track of counts for individual vis diff --git a/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java b/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java index d0f7bc50d33..498ad22106b 100644 --- a/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java +++ b/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java @@ -471,8 +471,9 @@ private void addHitsForFunction(String field, ASTFunctionNode node, HitListArith } try { - cv = AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(MarkingFunctionsFactory.createMarkingFunctions(), cvs.stream() - .map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))).collect(Collectors.toSet()))); + cv = AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(MarkingFunctionsFactory.createMarkingFunctions(), + cvs.stream().map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))) + .collect(Collectors.toSet()))); } catch (MarkingFunctions.Exception e) { log.error("Failed to combine column visibilities while generating HIT_TERM for phrase function for field [" + field + "]"); log.error("msg: ", e); diff --git a/warehouse/query-core/src/main/java/datawave/query/table/parser/EventKeyValueFactory.java b/warehouse/query-core/src/main/java/datawave/query/table/parser/EventKeyValueFactory.java index 1133e86a7ff..bc94602e268 100644 --- a/warehouse/query-core/src/main/java/datawave/query/table/parser/EventKeyValueFactory.java +++ b/warehouse/query-core/src/main/java/datawave/query/table/parser/EventKeyValueFactory.java @@ -40,7 +40,8 @@ public static EventKeyValue parse(Key key, Value value, Authorizations auths, Ma } protected static void parseColumnVisibility(EventKeyValue event, Key key) { - event.setMarkings(AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(new ColumnVisibility(key.getColumnVisibility())))); + event.setMarkings(AccessExpressionUtil + .toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(new ColumnVisibility(key.getColumnVisibility())))); } public static class EventKeyValue { diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountAggregatingIterator.java b/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountAggregatingIterator.java index 6a7ab2bbe6e..4ba85309b06 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountAggregatingIterator.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountAggregatingIterator.java @@ -154,8 +154,10 @@ public Object getAggregatedEvent() { private ColumnVisibility getCombinedColumnVisibility() { try { - return AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(markingFunctions, columnVisibilities.stream() - .map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))).collect(Collectors.toSet()))); + return AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(markingFunctions, + columnVisibilities.stream() + .map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))) + .collect(Collectors.toSet()))); } catch (Exception e) { log.error("Could not combine columnVisibilities for the count", e); return null; diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java index caf38688bad..3e0b3487505 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java @@ -509,8 +509,10 @@ public ColumnVisibility getColumnVisibility() { for (Text t : this.uniqueVisibilities) { columnVisibilities.add(new ColumnVisibility(t)); } - return AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(tupleMarkingFunctions, columnVisibilities.stream() - .map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))).collect(Collectors.toList()))); + return AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(tupleMarkingFunctions, + columnVisibilities.stream() + .map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))) + .collect(Collectors.toList()))); } catch (MarkingFunctions.Exception e) { logger.error("Could not create combined column visibility for the count", e); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java index 667c22068d6..2c53e873933 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java @@ -163,7 +163,8 @@ private EventBase _transform(Entry documentEntry) throws EmptyObje return output; } - protected EventBase buildResponse(Document document, Key documentKey, ColumnVisibility eventCV, String colf, String row, MarkingFunctions markingFunctions) { + protected EventBase buildResponse(Document document, Key documentKey, ColumnVisibility eventCV, String colf, String row, + MarkingFunctions markingFunctions) { Markings markings = AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(eventCV)); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java index 698eae50691..7b543f158f4 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java @@ -146,7 +146,8 @@ protected Collection buildFacets(Key documentKey, String f return myFields; } - protected FacetsBase buildResponse(Document document, Key documentKey, ColumnVisibility eventCV, String colf, String row, MarkingFunctions markingFunctions) { + protected FacetsBase buildResponse(Document document, Key documentKey, ColumnVisibility eventCV, String colf, String row, + MarkingFunctions markingFunctions) { FacetsBase facetedResponse = responseObjectFactory.getFacets(); diff --git a/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleField.java b/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleField.java index 64cb4030dae..6baec2b3eb9 100644 --- a/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleField.java +++ b/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleField.java @@ -1,5 +1,7 @@ package datawave.webservice.query.result.event; +import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; + import java.io.IOException; import java.io.Serializable; import java.util.HashMap; @@ -19,7 +21,6 @@ import datawave.data.type.Type; import datawave.marking.AccessExpressionMarkings; -import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.webservice.query.util.TypedValue; import io.protostuff.Input; @@ -27,8 +28,6 @@ import io.protostuff.Output; import io.protostuff.Schema; -import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; - /** * This object is contained inside of Event objects to describe name/value pairs of data in the Event. */ diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/keyword/TagCloudTestUtil.java b/warehouse/query-core/src/test/java/datawave/query/tables/keyword/TagCloudTestUtil.java index ccbd20fa6a4..5b1aa9732d4 100644 --- a/warehouse/query-core/src/test/java/datawave/query/tables/keyword/TagCloudTestUtil.java +++ b/warehouse/query-core/src/test/java/datawave/query/tables/keyword/TagCloudTestUtil.java @@ -5,13 +5,13 @@ import java.util.Map; import org.apache.accumulo.access.Access; +import org.apache.accumulo.access.examples.ParseExamples; import datawave.marking.AccessExpressionMarkings; import datawave.marking.Markings; import datawave.query.tables.keyword.transform.KeywordResultsTransformer; import datawave.webservice.result.keyword.DefaultTagCloud; import datawave.webservice.result.keyword.DefaultTagCloudEntry; -import org.apache.accumulo.access.examples.ParseExamples; /** * Utility for producing TagClouds and supporting objects for tests diff --git a/warehouse/query-core/src/test/java/datawave/query/transformer/FieldRenameTransformTest.java b/warehouse/query-core/src/test/java/datawave/query/transformer/FieldRenameTransformTest.java index 98e1c356b46..347d1d7afeb 100644 --- a/warehouse/query-core/src/test/java/datawave/query/transformer/FieldRenameTransformTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/transformer/FieldRenameTransformTest.java @@ -12,7 +12,6 @@ import org.apache.commons.collections.keyvalue.UnmodifiableMapEntry; import org.junit.Test; -import datawave.marking.MarkingFunctions; import datawave.query.Constants; import datawave.query.attributes.Attributes; import datawave.query.attributes.Document; From e44b837d5623ee8f2a2e5af044c97b095f72ef80 Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Thu, 2 Apr 2026 11:44:08 -0400 Subject: [PATCH 15/63] review feedback --- .../ColumnVisibilitySecurityMarking.java | 9 +- core/cached-results/pom.xml | 5 - .../cachedresults/CacheableQueryRowImpl.java | 7 +- .../CacheableQueryRowReader.java | 4 +- core/common-util/pom.xml | 4 - .../modification/MutableMetadataHandler.java | 4 +- core/utils/accumulo-utils/pom.xml | 5 - .../marking/AccessExpressionMarkings.java | 3 +- .../marking}/AccessExpressionUtil.java | 147 ++++++++++++++++-- .../datawave/marking/MarkingFunctions.java | 11 +- .../MarkingFunctionsDefaultCombineTest.java | 4 +- core/utils/metadata-utils/pom.xml | 5 - .../FrequencyMetadataAggregator.java | 2 +- microservices/services/accumulo/api/pom.xml | 2 +- .../response/objects/DefaultKey.java | 4 +- .../response/LookupResponseTest.java | 2 +- .../accumulo/admin/AdminService.java | 4 +- .../accumulo/lookup/LookupService.java | 4 +- microservices/services/dictionary/api/pom.xml | 2 +- .../dictionary/data/DefaultDescription.java | 2 +- .../DataDictionaryControllerLogic.java | 2 +- .../metadata/DefaultMetadataFieldScanner.java | 4 +- .../metadata/MetadataDescriptionsHelper.java | 2 +- .../services/modification/api/pom.xml | 2 +- .../querymetric/BaseQueryMetric.java | 6 +- .../querymetric/QueryMetricTest.java | 2 +- .../querymetric/QueryMetricOperations.java | 2 +- .../handler/ShardTableQueryMetricHandler.java | 2 +- .../querymetric/QueryMetricTestBase.java | 2 +- microservices/services/query/api/pom.xml | 2 +- .../ingest/config/RawRecordContainerImpl.java | 6 +- .../ingest/data/config/MarkingsHelper.java | 6 +- .../data/config/ingest/CompositeIngest.java | 2 +- .../data/config/ingest/VirtualIngest.java | 2 +- .../handler/atom/AtomDataTypeHandler.java | 2 +- .../atom/AtomErrorDataTypeHandler.java | 2 +- .../dateindex/DateIndexDataTypeHandler.java | 2 +- .../edge/ProtobufEdgeDataTypeHandler.java | 2 +- .../handler/edge/define/EdgeDataBundle.java | 2 +- .../handler/error/ErrorDataTypeHandler.java | 2 +- .../error/ErrorShardedDataTypeHandler.java | 2 +- .../mapreduce/handler/facet/FacetHandler.java | 2 +- .../handler/shard/ShardedDataTypeHandler.java | 2 +- .../ingest/mapreduce/SimpleRawRecord.java | 5 +- .../ingest/csv/mr/input/CSVRecordReader.java | 4 +- warehouse/keyword-common/pom.xml | 5 - .../util/keyword/DefaultTagCloudUtils.java | 4 +- .../java/datawave/util/keyword/TagCloud.java | 2 +- ...eldIndexCountingIteratorPerVisibility.java | 2 +- .../GlobalIndexDateSummaryIterator.java | 2 +- .../iterators/ResultCountingIterator.java | 2 +- .../datawave/query/attributes/Attribute.java | 2 +- .../query/attributes/AttributeBag.java | 2 +- .../datawave/query/attributes/Document.java | 2 +- .../query/common/grouping/GroupingUtils.java | 2 +- .../query/discovery/DiscoveryIterator.java | 2 +- .../query/discovery/DiscoveryTransformer.java | 2 +- .../query/discovery/TermInfoAggregation.java | 2 +- .../iterators/FieldIndexCountingIterator.java | 2 +- .../query/jexl/DatawaveInterpreter.java | 2 +- .../query/predicate/ValueToAttributes.java | 2 +- .../table/parser/EventKeyValueFactory.java | 2 +- .../parser/TermFrequencyKeyValueFactory.java | 2 +- .../shard/CountAggregatingIterator.java | 2 +- .../shard/CountResultPostprocessor.java | 2 +- .../shard/FieldIndexCountQueryLogic.java | 2 +- .../transformer/DocumentTransformer.java | 2 +- .../DocumentTransformerSupport.java | 7 +- .../transformer/EdgeQueryTransformer.java | 2 +- .../EdgeQueryTransformerSupport.java | 2 +- .../transformer/EventQueryTransformer.java | 2 +- .../EventQueryTransformerSupport.java | 2 +- .../query/transformer/FacetedTransformer.java | 2 +- .../FieldIndexCountQueryTransformer.java | 2 +- .../GroupingDocumentTransformer.java | 2 +- .../ShardIndexQueryTransformer.java | 2 +- .../ShardQueryCountTableTransformer.java | 2 +- .../transformer/TagCloudTransformer.java | 2 +- .../query/result/event/SimpleField.java | 2 +- .../common/grouping/DocumentGrouperTest.java | 5 +- .../tables/keyword/TagCloudTestUtil.java | 7 +- .../metrics/ShardTableQueryMetricHandler.java | 2 +- 82 files changed, 238 insertions(+), 153 deletions(-) rename core/{common-util/src/main/java/datawave/core/common/util => utils/accumulo-utils/src/main/java/datawave/marking}/AccessExpressionUtil.java (53%) diff --git a/core/base-rest-responses/src/main/java/datawave/marking/ColumnVisibilitySecurityMarking.java b/core/base-rest-responses/src/main/java/datawave/marking/ColumnVisibilitySecurityMarking.java index 8c1a0eab638..b5576800956 100644 --- a/core/base-rest-responses/src/main/java/datawave/marking/ColumnVisibilitySecurityMarking.java +++ b/core/base-rest-responses/src/main/java/datawave/marking/ColumnVisibilitySecurityMarking.java @@ -1,5 +1,7 @@ package datawave.marking; +import static datawave.marking.AccessExpressionMarkings.ACCESS; + import java.util.List; import java.util.Map; @@ -9,10 +11,7 @@ import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlAttribute; -import org.apache.accumulo.access.Access; import org.apache.accumulo.access.AccessExpression; -import org.apache.accumulo.access.examples.ParseExamples; -import org.apache.accumulo.access.impl.AccessExpressionImpl; import com.google.common.base.Preconditions; @@ -22,8 +21,6 @@ public class ColumnVisibilitySecurityMarking implements SecurityMarking { public static final String VISIBILITY_MARKING = "columnVisibility"; - private static final Access ACCESS = ParseExamples.ACCESS; - @XmlAttribute(name = "columnVisibility") private String columnVisibility = null; @@ -51,7 +48,7 @@ public void setColumnVisibility(String columnVisibility) { @Override public AccessExpression toAccessExpression() { if (columnVisibility == null || columnVisibility.isEmpty()) { - return AccessExpressionImpl.EMPTY; + return ACCESS.newExpression(""); } return ACCESS.newExpression(columnVisibility); } diff --git a/core/cached-results/pom.xml b/core/cached-results/pom.xml index 6e53d7e5cc9..c9e7a44f4f3 100644 --- a/core/cached-results/pom.xml +++ b/core/cached-results/pom.xml @@ -15,11 +15,6 @@ gov.nsa.datawave.core base-rest-responses - - gov.nsa.datawave.core - datawave-core-common-util - ${project.version} - gov.nsa.datawave.core type-utils diff --git a/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java b/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java index 27dde06fe60..0a1f789345f 100644 --- a/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java +++ b/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java @@ -1,5 +1,7 @@ package datawave.core.query.cachedresults; +import static datawave.marking.AccessExpressionMarkings.ACCESS; + import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -9,16 +11,15 @@ import java.util.Set; import java.util.TreeSet; -import org.apache.accumulo.access.impl.AccessExpressionImpl; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.collect.Sets; -import datawave.core.common.util.AccessExpressionUtil; import datawave.data.type.Type; import datawave.marking.AccessExpressionMarkings; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.webservice.query.cachedresults.CacheableQueryRow; @@ -28,7 +29,7 @@ public class CacheableQueryRowImpl extends CacheableQueryRow implements ObjectSizeOf { private static final Logger LOGGER = LoggerFactory.getLogger(CacheableQueryRowImpl.class); - private static final Markings EMPTY_MARKINGS = AccessExpressionMarkings.builder().accessExpression(AccessExpressionImpl.EMPTY).build(); + private static final Markings EMPTY_MARKINGS = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression("")).build(); private String user = null; private String queryId = null; diff --git a/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowReader.java b/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowReader.java index b60c8bcf6f3..d7d27746774 100644 --- a/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowReader.java +++ b/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowReader.java @@ -1,6 +1,6 @@ package datawave.core.query.cachedresults; -import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; +import static datawave.marking.AccessExpressionMarkings.ACCESS; import java.sql.ResultSet; import java.sql.ResultSetMetaData; @@ -14,7 +14,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import datawave.core.common.util.AccessExpressionUtil; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.webservice.query.cachedresults.CacheableQueryRow; diff --git a/core/common-util/pom.xml b/core/common-util/pom.xml index 5461acea289..5d815e50965 100644 --- a/core/common-util/pom.xml +++ b/core/common-util/pom.xml @@ -13,10 +13,6 @@ - - gov.nsa.datawave.core - accumulo-utils - gov.nsa.datawave.core datawave-core-connection-pool diff --git a/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java b/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java index a89cb82151b..dbb003917b1 100644 --- a/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java +++ b/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java @@ -1,6 +1,6 @@ package datawave.modification; -import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; +import static datawave.marking.AccessExpressionMarkings.ACCESS; import java.util.ArrayList; import java.util.Arrays; @@ -47,12 +47,12 @@ import com.google.common.collect.Multimap; import datawave.core.common.connection.AccumuloConnectionFactory; -import datawave.core.common.util.AccessExpressionUtil; import datawave.core.iterators.FieldIndexDocumentFilter; import datawave.data.ColumnFamilyConstants; import datawave.data.type.Type; import datawave.ingest.protobuf.Uid; import datawave.ingest.protobuf.Uid.List.Builder; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.microservice.query.DefaultQueryParameters; diff --git a/core/utils/accumulo-utils/pom.xml b/core/utils/accumulo-utils/pom.xml index 03d9a346ded..a09c9775a48 100644 --- a/core/utils/accumulo-utils/pom.xml +++ b/core/utils/accumulo-utils/pom.xml @@ -67,11 +67,6 @@ accumulo-access-core 1.0.0-java-11 - - org.apache.accumulo - accumulo-access-examples - 1.0.0-java-11 - org.apache.commons commons-collections4 diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java index 63ae8da07dc..bd34e94e51a 100644 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java @@ -2,7 +2,6 @@ import org.apache.accumulo.access.Access; import org.apache.accumulo.access.AccessExpression; -import org.apache.accumulo.access.examples.ParseExamples; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonCreator; @@ -15,8 +14,8 @@ @Builder @JsonAutoDetect(getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE) public class AccessExpressionMarkings implements Markings { + public static final Access ACCESS = Access.builder().build(); - private static final Access ACCESS = ParseExamples.ACCESS; public static final String COLUMN_VISIBILITY_KEY = "columnVisibility"; private final AccessExpression accessExpression; diff --git a/core/common-util/src/main/java/datawave/core/common/util/AccessExpressionUtil.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionUtil.java similarity index 53% rename from core/common-util/src/main/java/datawave/core/common/util/AccessExpressionUtil.java rename to core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionUtil.java index f9021717fad..3835bf72d19 100644 --- a/core/common-util/src/main/java/datawave/core/common/util/AccessExpressionUtil.java +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionUtil.java @@ -1,21 +1,21 @@ -package datawave.core.common.util; +package datawave.marking; import static java.nio.charset.StandardCharsets.UTF_8; +import static datawave.marking.AccessExpressionMarkings.ACCESS; +import static org.apache.accumulo.access.ParsedAccessExpression.ExpressionType.AND; +import static org.apache.accumulo.access.ParsedAccessExpression.ExpressionType.AUTHORIZATION; + import java.util.Collection; import java.util.List; +import java.util.SortedSet; +import java.util.TreeSet; import org.apache.accumulo.access.Access; import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.access.ParsedAccessExpression; -import org.apache.accumulo.access.examples.ParseExamples; -import org.apache.accumulo.access.impl.AccessExpressionImpl; import org.apache.accumulo.core.security.ColumnVisibility; -import datawave.marking.AccessExpressionMarkings; -import datawave.marking.MarkingFunctions; -import datawave.marking.Markings; - /** * Utility for bridging between Accumulo's {@link ColumnVisibility} (from accumulo-core) and {@link AccessExpression} (from accumulo-access). *

@@ -24,9 +24,7 @@ */ public final class AccessExpressionUtil { - private static final Access ACCESS = ParseExamples.ACCESS; - - private static final AccessExpression EMPTY_EXPRESSION = AccessExpressionImpl.EMPTY; + private static final AccessExpression EMPTY_EXPRESSION = ACCESS.newExpression(""); private AccessExpressionUtil() {} @@ -128,7 +126,7 @@ public static AccessExpression normalize(AccessExpression expression) { return expression; } ParsedAccessExpression parsed = ACCESS.newParsedExpression(expr); - return ACCESS.newExpression(ParseExamples.normalize(parsed).expression); + return ACCESS.newExpression(normalize(parsed).expression); } /** @@ -192,4 +190,131 @@ public static Markings mergeMarkings(MarkingFunctions markingFunctions, Ma } return markings1; } + + /** + * As part of normalizing access expression this class is used to sort and dedupe sub-expressions in a tree set. + */ + public static class NormalizedExpression implements Comparable { + public final String expression; + public final ParsedAccessExpression.ExpressionType type; + + NormalizedExpression(String expression, ParsedAccessExpression.ExpressionType type) { + this.expression = expression; + this.type = type; + } + + // determines the sort order of different kinds of subexpressions. + private static int typeOrder(ParsedAccessExpression.ExpressionType type) { + switch (type) { + case AUTHORIZATION: + return 1; + case OR: + return 2; + case AND: + return 3; + case EMPTY: + default: + throw new IllegalArgumentException("Unexpected type " + type); + } + } + + @Override + public int compareTo(NormalizedExpression o) { + // Changing this comparator would significantly change how expressions are normalized. + int cmp = typeOrder(type) - typeOrder(o.type); + if (cmp == 0) { + if (type == AUTHORIZATION) { + // sort based on the unquoted and unescaped form of the authorization + cmp = ACCESS.unquote(expression).compareTo(ACCESS.unquote(o.expression)); + } else { + cmp = expression.compareTo(o.expression); + } + + } + return cmp; + } + + @Override + public boolean equals(Object o) { + return this == o || (o instanceof NormalizedExpression && compareTo((NormalizedExpression) o) == 0); + } + + @Override + public int hashCode() { + return expression.hashCode(); + } + } + + /** + * This method helps with the flattening aspect of normalization by recursing down as far as possible the parse tree in the case when the expression type is + * the same. As long as the type is the same in the sub expression, keep using the same set. + */ + public static void flatten(ParsedAccessExpression.ExpressionType parentType, ParsedAccessExpression parsed, + SortedSet normalizedExpressions) { + if (parsed.getType() == parentType) { + for (var child : parsed.getChildren()) { + flatten(parentType, child, normalizedExpressions); + } + } else { + // The type changed, so start again on the subexpression. + normalizedExpressions.add(normalize(parsed)); + } + } + + /** + *

+ * For a given access expression this example will deduplicate, sort, flatten, and remove unneeded parentheses or quotes in the expressions. The following + * list gives examples of what each normalization step does. + * + *

    + *
  • As an example of flattening, the expression {@code A&(B&C)} flattens to {@code + * A&B&C}.
    • + *
  • As an example of sorting, the expression {@code (Z&Y)|(C&B)} sorts to {@code + * (B&C)|(Y&Z)}
    • + *
  • As an example of deduplication, the expression {@code X&Y&X} normalizes to {@code X&Y}
    • + *
  • As an example of unneeded quotes, the expression {@code "ABC"&"XYZ"} normalizes to {@code ABC&XYZ}
    • + *
  • As an example of unneeded parentheses, the expression {@code (((ABC)|(XYZ)))} normalizes to {@code ABC|XYZ}
    • + *
+ * + *

+ * This algorithm attempts to have the same behavior as the one in the Accumulo 2.1 ColumnVisibility class. However the implementation is very different. + */ + public static NormalizedExpression normalize(ParsedAccessExpression parsed) { + if (parsed.getType() == AUTHORIZATION) { + // If the authorization is quoted and it does not need to be quoted then the following two + // lines will remove the unnecessary quoting. + String unquoted = ACCESS.unquote(parsed.getExpression()); + String quoted = ACCESS.quote(unquoted); + return new NormalizedExpression(quoted, parsed.getType()); + } else { + // The tree set does the work of sorting and deduplicating sub expressions. + TreeSet normalizedChildren = new TreeSet<>(); + for (var child : parsed.getChildren()) { + flatten(parsed.getType(), child, normalizedChildren); + } + + if (normalizedChildren.size() == 1) { + return normalizedChildren.first(); + } else { + String operator = parsed.getType() == AND ? "&" : "|"; + String sep = ""; + + StringBuilder builder = new StringBuilder(); + + for (var child : normalizedChildren) { + builder.append(sep); + if (child.type == AUTHORIZATION) { + builder.append(child.expression); + } else { + builder.append("("); + builder.append(child.expression); + builder.append(")"); + } + sep = operator; + } + + return new NormalizedExpression(builder.toString(), parsed.getType()); + } + } + } } diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java index b26bea712af..7142f6ed2d6 100644 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java @@ -1,15 +1,14 @@ package datawave.marking; +import static datawave.marking.AccessExpressionMarkings.ACCESS; + import java.util.Collection; import java.util.HashSet; import java.util.Set; import java.util.stream.Collectors; -import org.apache.accumulo.access.Access; import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.access.ParsedAccessExpression; -import org.apache.accumulo.access.examples.ParseExamples; -import org.apache.accumulo.access.impl.AccessExpressionImpl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.support.ClassPathXmlApplicationContext; @@ -49,8 +48,8 @@ public Exception(Throwable cause) { } class Default implements MarkingFunctions { - private static final Access ACCESS = ParseExamples.ACCESS; - private static final AccessExpressionMarkings EMPTY_MARKINGS = AccessExpressionMarkings.builder().accessExpression(AccessExpressionImpl.EMPTY).build(); + + private static final AccessExpressionMarkings EMPTY_MARKINGS = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression("")).build(); @Override public AccessExpressionMarkings combine(Collection markings) { @@ -80,7 +79,7 @@ public AccessExpressionMarkings combine(Collection mar // normalize the Parsed Expression and then return a copy without the parse tree // FIXME: Using a beta of the accumulo-access API so temporarily using the normalize function from the example code - return AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression(ParseExamples.normalize(expression).expression)).build(); + return AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression(AccessExpressionUtil.normalize(expression).expression)).build(); } } diff --git a/core/utils/accumulo-utils/src/test/java/datawave/marking/MarkingFunctionsDefaultCombineTest.java b/core/utils/accumulo-utils/src/test/java/datawave/marking/MarkingFunctionsDefaultCombineTest.java index 2ff91e3f71c..45af0b2a48e 100644 --- a/core/utils/accumulo-utils/src/test/java/datawave/marking/MarkingFunctionsDefaultCombineTest.java +++ b/core/utils/accumulo-utils/src/test/java/datawave/marking/MarkingFunctionsDefaultCombineTest.java @@ -1,15 +1,13 @@ package datawave.marking; +import static datawave.marking.AccessExpressionMarkings.ACCESS; import static org.junit.jupiter.api.Assertions.assertEquals; import java.util.List; -import org.apache.accumulo.access.Access; -import org.apache.accumulo.access.examples.ParseExamples; import org.junit.jupiter.api.Test; public class MarkingFunctionsDefaultCombineTest { - public static final Access ACCESS = ParseExamples.ACCESS; @Test public void testCombineAnds() throws MarkingFunctions.Exception { diff --git a/core/utils/metadata-utils/pom.xml b/core/utils/metadata-utils/pom.xml index 279ca3488df..d089c11e3f6 100644 --- a/core/utils/metadata-utils/pom.xml +++ b/core/utils/metadata-utils/pom.xml @@ -59,11 +59,6 @@ gov.nsa.datawave.core common-utils - - gov.nsa.datawave.core - datawave-core-common-util - ${project.version} - gov.nsa.datawave.core type-utils diff --git a/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java b/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java index 256f12182ba..ea5004f2c14 100644 --- a/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java +++ b/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java @@ -30,8 +30,8 @@ import com.google.common.base.Splitter; -import datawave.core.common.util.AccessExpressionUtil; import datawave.iterator.ReducingIterator; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.query.model.DateFrequencyMap; import datawave.util.StringUtils; diff --git a/microservices/services/accumulo/api/pom.xml b/microservices/services/accumulo/api/pom.xml index 1c823e594d2..ea9aa40fc54 100644 --- a/microservices/services/accumulo/api/pom.xml +++ b/microservices/services/accumulo/api/pom.xml @@ -19,7 +19,7 @@ http://webservice.datawave.nsa/v1 1.17.1 - 7.33.1 + 7.38.0-SNAPSHOT 2.14.3 2.3.3 diff --git a/microservices/services/accumulo/api/src/main/java/datawave/webservice/response/objects/DefaultKey.java b/microservices/services/accumulo/api/src/main/java/datawave/webservice/response/objects/DefaultKey.java index b720dba89f7..83db2a29b8f 100644 --- a/microservices/services/accumulo/api/src/main/java/datawave/webservice/response/objects/DefaultKey.java +++ b/microservices/services/accumulo/api/src/main/java/datawave/webservice/response/objects/DefaultKey.java @@ -2,8 +2,6 @@ import static java.nio.charset.StandardCharsets.UTF_8; -import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; - import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlElement; @@ -115,7 +113,7 @@ public void setColQual(String colQual) { public void setColumnVisibility(String colVis) { this.columnVisibility = (colVis == null) ? new TypedValue("") : new TypedValue(colVis); String expr = this.columnVisibility.getValue().toString(); - this.markings = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression(expr)).build(); + this.markings = AccessExpressionMarkings.builder().accessExpression(AccessExpressionMarkings.ACCESS.newExpression(expr)).build(); } public void setTimestamp(long timestamp) { diff --git a/microservices/services/accumulo/api/src/test/java/datawave/webservice/response/LookupResponseTest.java b/microservices/services/accumulo/api/src/test/java/datawave/webservice/response/LookupResponseTest.java index 3c0d318fb32..d2187ef1768 100644 --- a/microservices/services/accumulo/api/src/test/java/datawave/webservice/response/LookupResponseTest.java +++ b/microservices/services/accumulo/api/src/test/java/datawave/webservice/response/LookupResponseTest.java @@ -1,6 +1,6 @@ package datawave.webservice.response; -import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; +import static datawave.marking.AccessExpressionMarkings.ACCESS; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; diff --git a/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/admin/AdminService.java b/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/admin/AdminService.java index f2ec378d63d..662f638d00f 100644 --- a/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/admin/AdminService.java +++ b/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/admin/AdminService.java @@ -2,7 +2,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; -import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; +import static datawave.marking.AccessExpressionMarkings.ACCESS; import java.util.ArrayList; import java.util.HashMap; @@ -43,7 +43,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; -import datawave.core.common.util.AccessExpressionUtil; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.webservice.query.util.OptionallyEncodedString; diff --git a/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/lookup/LookupService.java b/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/lookup/LookupService.java index ded3233dc5b..f1c5faffcd2 100644 --- a/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/lookup/LookupService.java +++ b/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/lookup/LookupService.java @@ -2,7 +2,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; -import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; +import static datawave.marking.AccessExpressionMarkings.ACCESS; import java.util.ArrayList; import java.util.Arrays; @@ -36,7 +36,7 @@ import com.google.common.base.Preconditions; import datawave.accumulo.util.security.UserAuthFunctions; -import datawave.core.common.util.AccessExpressionUtil; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.marking.SecurityMarking; diff --git a/microservices/services/dictionary/api/pom.xml b/microservices/services/dictionary/api/pom.xml index 60ab10091fb..4244cc91c95 100644 --- a/microservices/services/dictionary/api/pom.xml +++ b/microservices/services/dictionary/api/pom.xml @@ -19,7 +19,7 @@ http://webservice.datawave.nsa/v1 3.20.0 - 7.33.1 + 7.38.0-SNAPSHOT 31.1-jre 2.3.3 1.6.2 diff --git a/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java b/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java index dbc3f7fd792..690ae6aa221 100644 --- a/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java +++ b/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java @@ -1,6 +1,6 @@ package datawave.webservice.dictionary.data; -import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; +import static datawave.marking.AccessExpressionMarkings.ACCESS; import java.io.IOException; import java.io.Serializable; diff --git a/microservices/services/dictionary/service/src/main/java/datawave/microservice/dictionary/DataDictionaryControllerLogic.java b/microservices/services/dictionary/service/src/main/java/datawave/microservice/dictionary/DataDictionaryControllerLogic.java index 1ce6203253f..55d93d5348d 100644 --- a/microservices/services/dictionary/service/src/main/java/datawave/microservice/dictionary/DataDictionaryControllerLogic.java +++ b/microservices/services/dictionary/service/src/main/java/datawave/microservice/dictionary/DataDictionaryControllerLogic.java @@ -1,6 +1,6 @@ package datawave.microservice.dictionary; -import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; +import static datawave.marking.AccessExpressionMarkings.ACCESS; import java.util.Arrays; import java.util.Collection; diff --git a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java index f0f51742da7..9dcf9eeb3c2 100644 --- a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java +++ b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java @@ -1,6 +1,6 @@ package datawave.microservice.metadata; -import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; +import static datawave.marking.AccessExpressionMarkings.ACCESS; import java.time.Instant; import java.time.ZoneId; @@ -24,8 +24,8 @@ import com.google.common.collect.Maps; -import datawave.core.common.util.AccessExpressionUtil; import datawave.data.ColumnFamilyConstants; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.microservice.Connection; import datawave.microservice.dictionary.config.ResponseObjectFactory; diff --git a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java index de10da1ede8..f221923ecef 100644 --- a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java +++ b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java @@ -30,8 +30,8 @@ import com.google.common.collect.Multimaps; import com.google.common.collect.SetMultimap; -import datawave.core.common.util.AccessExpressionUtil; import datawave.data.ColumnFamilyConstants; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.microservice.dictionary.config.ResponseObjectFactory; diff --git a/microservices/services/modification/api/pom.xml b/microservices/services/modification/api/pom.xml index c243e3a75ef..eeaf6ad657d 100644 --- a/microservices/services/modification/api/pom.xml +++ b/microservices/services/modification/api/pom.xml @@ -18,7 +18,7 @@ 2.1.4-5792fed3-bulkv2 - 7.33.1 + 7.38.0-SNAPSHOT 31.1-jre diff --git a/microservices/services/query-metric/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java b/microservices/services/query-metric/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java index fec797334a1..c6cd45d038b 100644 --- a/microservices/services/query-metric/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java +++ b/microservices/services/query-metric/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java @@ -1,5 +1,7 @@ package datawave.microservice.querymetric; +import static datawave.marking.AccessExpressionMarkings.ACCESS; + import java.io.IOException; import java.io.InputStream; import java.io.Serializable; @@ -24,9 +26,7 @@ import javax.xml.bind.annotation.XmlElementWrapper; import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; -import org.apache.accumulo.access.Access; import org.apache.accumulo.access.AccessExpression; -import org.apache.accumulo.access.examples.ParseExamples; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.builder.CompareToBuilder; import org.apache.commons.lang3.builder.EqualsBuilder; @@ -1168,8 +1168,6 @@ public void setPredictions(Set predictions) { this.predictions = predictions; } - private static final Access ACCESS = ParseExamples.ACCESS; - @Override public void setMarkings(Markings markings) { if (markings == null || markings.isEmpty()) { diff --git a/microservices/services/query-metric/api/src/test/java/datawave/microservice/querymetric/QueryMetricTest.java b/microservices/services/query-metric/api/src/test/java/datawave/microservice/querymetric/QueryMetricTest.java index e100f3741b8..dc2a09df0e9 100644 --- a/microservices/services/query-metric/api/src/test/java/datawave/microservice/querymetric/QueryMetricTest.java +++ b/microservices/services/query-metric/api/src/test/java/datawave/microservice/querymetric/QueryMetricTest.java @@ -1,7 +1,7 @@ package datawave.microservice.querymetric; +import static datawave.marking.AccessExpressionMarkings.ACCESS; import static datawave.webservice.query.exception.DatawaveErrorCode.NO_QUERY_RESULTS_FOUND; -import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; diff --git a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java index 28a04c760ea..02de938d7d6 100644 --- a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java +++ b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java @@ -53,7 +53,7 @@ import com.hazelcast.map.IMap; import com.hazelcast.spring.cache.HazelcastCacheManager; -import datawave.core.common.util.AccessExpressionUtil; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.microservice.authorization.user.DatawaveUserDetails; import datawave.microservice.querymetric.config.QueryMetricTransportType; diff --git a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java index 5041a653288..0452d37ffe1 100644 --- a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java +++ b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java @@ -48,7 +48,6 @@ import com.google.common.collect.Multimap; import datawave.core.common.connection.AccumuloConnectionFactory; -import datawave.core.common.util.AccessExpressionUtil; import datawave.core.query.util.QueryUtil; import datawave.data.hash.UID; import datawave.data.hash.UIDBuilder; @@ -63,6 +62,7 @@ import datawave.ingest.mapreduce.job.BulkIngestKey; import datawave.ingest.protobuf.Uid; import datawave.ingest.table.config.TableConfigHelper; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.microservice.authorization.user.DatawaveUserDetails; diff --git a/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java b/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java index 2af6b5055a3..1e422b8e02b 100644 --- a/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java +++ b/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java @@ -1,8 +1,8 @@ package datawave.microservice.querymetric; +import static datawave.marking.AccessExpressionMarkings.ACCESS; import static datawave.microservice.querymetric.config.HazelcastMetricCacheConfiguration.INCOMING_METRICS; import static datawave.security.authorization.DatawaveUser.UserType.USER; -import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; diff --git a/microservices/services/query/api/pom.xml b/microservices/services/query/api/pom.xml index 4e137af4e36..49e2241a85e 100644 --- a/microservices/services/query/api/pom.xml +++ b/microservices/services/query/api/pom.xml @@ -20,7 +20,7 @@ http://webservice.datawave.nsa/v1 4.3 3.20.0 - 7.33.1 + 7.38.0-SNAPSHOT 31.1-jre 2.14.3 diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/config/RawRecordContainerImpl.java b/warehouse/ingest-core/src/main/java/datawave/ingest/config/RawRecordContainerImpl.java index 1d335f7ec2c..b2d62853beb 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/config/RawRecordContainerImpl.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/config/RawRecordContainerImpl.java @@ -1,5 +1,7 @@ package datawave.ingest.config; +import static datawave.marking.AccessExpressionMarkings.ACCESS; + import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; @@ -15,9 +17,7 @@ import java.util.Set; import java.util.concurrent.ConcurrentSkipListSet; -import org.apache.accumulo.access.Access; import org.apache.accumulo.access.AccessExpression; -import org.apache.accumulo.access.examples.ParseExamples; import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.ToStringBuilder; @@ -83,8 +83,6 @@ public class RawRecordContainerImpl implements Writable, Configurable, RawRecord private Object auxData = null; private Map auxMap = null; - private static final Access ACCESS = ParseExamples.ACCESS; - // RawRecordContainer support Markings securityMarkings = null; diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/MarkingsHelper.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/MarkingsHelper.java index 7558a088d2c..a28ce841048 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/MarkingsHelper.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/MarkingsHelper.java @@ -1,11 +1,11 @@ package datawave.ingest.data.config; +import static datawave.marking.AccessExpressionMarkings.ACCESS; + import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; -import org.apache.accumulo.access.Access; -import org.apache.accumulo.access.examples.ParseExamples; import org.apache.hadoop.conf.Configuration; import datawave.ingest.data.Type; @@ -55,8 +55,6 @@ public interface MarkingsHelper { */ class NoOp implements MarkingsHelper { - private static final Access ACCESS = ParseExamples.ACCESS; - private Map> fieldMarkingMap = new HashMap<>(); private Markings defaultMarkings = null; diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CompositeIngest.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CompositeIngest.java index 4dc8702a0cd..689c987dc16 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CompositeIngest.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CompositeIngest.java @@ -1,6 +1,6 @@ package datawave.ingest.data.config.ingest; -import static datawave.core.common.util.AccessExpressionUtil.mergeMarkings; +import static datawave.marking.AccessExpressionUtil.mergeMarkings; import java.util.ArrayList; import java.util.Arrays; diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java index 03dd69cee04..4baf799f293 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java @@ -1,6 +1,6 @@ package datawave.ingest.data.config.ingest; -import static datawave.core.common.util.AccessExpressionUtil.mergeMarkings; +import static datawave.marking.AccessExpressionUtil.mergeMarkings; import java.util.ArrayList; import java.util.Arrays; diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomDataTypeHandler.java index 40e52b5b545..0f2f3ae4f8b 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomDataTypeHandler.java @@ -19,7 +19,6 @@ import com.google.common.base.Strings; import com.google.common.collect.Multimap; -import datawave.core.common.util.AccessExpressionUtil; import datawave.ingest.data.RawRecordContainer; import datawave.ingest.data.Type; import datawave.ingest.data.TypeRegistry; @@ -30,6 +29,7 @@ import datawave.ingest.mapreduce.job.BulkIngestKey; import datawave.ingest.mapreduce.job.writer.ContextWriter; import datawave.ingest.metadata.RawRecordMetadata; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.util.StringUtils; diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomErrorDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomErrorDataTypeHandler.java index b6c22b5d336..24612ed07f5 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomErrorDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomErrorDataTypeHandler.java @@ -17,7 +17,6 @@ import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; -import datawave.core.common.util.AccessExpressionUtil; import datawave.data.hash.UID; import datawave.ingest.config.IngestConfigurationFactory; import datawave.ingest.data.RawDataErrorNames; @@ -30,6 +29,7 @@ import datawave.ingest.mapreduce.handler.error.ErrorShardedDataTypeHandler; import datawave.ingest.mapreduce.job.BulkIngestKey; import datawave.ingest.mapreduce.job.writer.ContextWriter; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/dateindex/DateIndexDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/dateindex/DateIndexDataTypeHandler.java index 232b1726cd3..545fe5da54c 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/dateindex/DateIndexDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/dateindex/DateIndexDataTypeHandler.java @@ -28,7 +28,6 @@ import com.google.common.collect.Multimap; import datawave.core.common.logging.ThreadConfigurableLogger; -import datawave.core.common.util.AccessExpressionUtil; import datawave.data.normalizer.DateNormalizer; import datawave.ingest.data.RawRecordContainer; import datawave.ingest.data.Type; @@ -40,6 +39,7 @@ import datawave.ingest.mapreduce.job.BulkIngestKey; import datawave.ingest.metadata.RawRecordMetadata; import datawave.ingest.table.aggregator.DateIndexDateAggregator; +import datawave.marking.AccessExpressionUtil; /** *

diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDataTypeHandler.java index b0e609942ab..1b9b6459f7d 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDataTypeHandler.java @@ -47,7 +47,6 @@ import com.google.common.hash.Hashing; import com.google.common.hash.PrimitiveSink; -import datawave.core.common.util.AccessExpressionUtil; import datawave.data.normalizer.DateNormalizer; import datawave.edge.util.EdgeKey; import datawave.edge.util.EdgeKey.EDGE_FORMAT; @@ -77,6 +76,7 @@ import datawave.ingest.metadata.RawRecordMetadata; import datawave.ingest.table.config.LoadDateTableConfigHelper; import datawave.ingest.time.Now; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.metadata.protobuf.EdgeMetadata; diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/EdgeDataBundle.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/EdgeDataBundle.java index df5c9512caa..a59959014f4 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/EdgeDataBundle.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/EdgeDataBundle.java @@ -1,6 +1,6 @@ package datawave.ingest.mapreduce.handler.edge.define; -import static datawave.core.common.util.AccessExpressionUtil.mergeMarkings; +import static datawave.marking.AccessExpressionUtil.mergeMarkings; import java.io.IOException; import java.util.Calendar; diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorDataTypeHandler.java index 33a033df521..a5553e4549d 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorDataTypeHandler.java @@ -20,7 +20,6 @@ import com.google.common.collect.Multimap; -import datawave.core.common.util.AccessExpressionUtil; import datawave.data.hash.UID; import datawave.data.hash.UIDBuilder; import datawave.ingest.config.IngestConfiguration; @@ -37,6 +36,7 @@ import datawave.ingest.mapreduce.job.BulkIngestKey; import datawave.ingest.mapreduce.job.writer.ContextWriter; import datawave.ingest.metadata.RawRecordMetadata; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.util.TextUtil; import datawave.util.time.DateHelper; diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java index a8210eac01e..5c406337210 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java @@ -24,7 +24,6 @@ import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; -import datawave.core.common.util.AccessExpressionUtil; import datawave.data.hash.UID; import datawave.ingest.config.IngestConfiguration; import datawave.ingest.config.IngestConfigurationFactory; @@ -44,6 +43,7 @@ import datawave.ingest.mapreduce.handler.shard.ShardedDataTypeHandler; import datawave.ingest.mapreduce.job.BulkIngestKey; import datawave.ingest.mapreduce.job.writer.ContextWriter; +import datawave.marking.AccessExpressionUtil; /** * Handler that take events with processing errors or fatal errors and dumps them into a processing error table. This table will be used for subsequent diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/facet/FacetHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/facet/FacetHandler.java index 74cd2c8db39..d205e84f6b2 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/facet/FacetHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/facet/FacetHandler.java @@ -35,7 +35,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Multimap; -import datawave.core.common.util.AccessExpressionUtil; import datawave.ingest.data.RawRecordContainer; import datawave.ingest.data.Type; import datawave.ingest.data.TypeRegistry; @@ -48,6 +47,7 @@ import datawave.ingest.mapreduce.job.BulkIngestKey; import datawave.ingest.mapreduce.job.writer.ContextWriter; import datawave.ingest.metadata.RawRecordMetadata; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.util.time.DateHelper; diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java index 1cecd7329f4..ff27490482f 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java @@ -34,7 +34,6 @@ import com.google.common.hash.BloomFilter; import datawave.core.common.logging.ThreadConfigurableLogger; -import datawave.core.common.util.AccessExpressionUtil; import datawave.ingest.config.IngestConfiguration; import datawave.ingest.config.IngestConfigurationFactory; import datawave.ingest.data.RawRecordContainer; @@ -52,6 +51,7 @@ import datawave.ingest.util.BloomFilterUtil; import datawave.ingest.util.BloomFilterWrapper; import datawave.ingest.util.DiskSpaceStarvationStrategy; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.query.model.Direction; diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/SimpleRawRecord.java b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/SimpleRawRecord.java index c3de6458b85..adfc53f201c 100644 --- a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/SimpleRawRecord.java +++ b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/SimpleRawRecord.java @@ -1,5 +1,7 @@ package datawave.ingest.mapreduce; +import static datawave.marking.AccessExpressionMarkings.ACCESS; + import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; @@ -10,9 +12,7 @@ import java.util.HashMap; import java.util.Map; -import org.apache.accumulo.access.Access; import org.apache.accumulo.access.AccessExpression; -import org.apache.accumulo.access.examples.ParseExamples; import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.hadoop.io.Writable; @@ -31,7 +31,6 @@ */ public class SimpleRawRecord implements RawRecordContainer, Writable { - private static final Access ACCESS = ParseExamples.ACCESS; private UIDBuilder uidBuilder = HashUID.builder(); private Markings securityMarkings = null; diff --git a/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java b/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java index 3e6bb55905b..0214745bb89 100644 --- a/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java +++ b/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java @@ -1,6 +1,6 @@ package datawave.ingest.csv.mr.input; -import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; +import static datawave.marking.AccessExpressionMarkings.ACCESS; import java.io.IOException; import java.util.Map; @@ -18,7 +18,6 @@ import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; -import datawave.core.common.util.AccessExpressionUtil; import datawave.data.hash.UID; import datawave.ingest.config.IngestConfiguration; import datawave.ingest.config.IngestConfigurationFactory; @@ -31,6 +30,7 @@ import datawave.ingest.metadata.id.MetadataIdParser; import datawave.ingest.validation.EventValidator; import datawave.marking.AccessExpressionMarkings; +import datawave.marking.AccessExpressionUtil; public class CSVRecordReader extends CSVReaderBase implements EventFixer { diff --git a/warehouse/keyword-common/pom.xml b/warehouse/keyword-common/pom.xml index f3a2b18f4b4..28e13e0f316 100644 --- a/warehouse/keyword-common/pom.xml +++ b/warehouse/keyword-common/pom.xml @@ -21,10 +21,5 @@ datawave-core ${project.version} - - gov.nsa.datawave.core - datawave-core-common-util - ${project.version} - diff --git a/warehouse/keyword-common/src/main/java/datawave/util/keyword/DefaultTagCloudUtils.java b/warehouse/keyword-common/src/main/java/datawave/util/keyword/DefaultTagCloudUtils.java index 233f38fa68d..417643cab3c 100644 --- a/warehouse/keyword-common/src/main/java/datawave/util/keyword/DefaultTagCloudUtils.java +++ b/warehouse/keyword-common/src/main/java/datawave/util/keyword/DefaultTagCloudUtils.java @@ -1,6 +1,6 @@ package datawave.util.keyword; -import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; +import static datawave.marking.AccessExpressionMarkings.ACCESS; import java.io.Serializable; import java.util.Collection; @@ -14,7 +14,7 @@ import org.apache.accumulo.access.AccessExpression; import org.apache.commons.lang3.StringUtils; -import datawave.core.common.util.AccessExpressionUtil; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; diff --git a/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloud.java b/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloud.java index cd36a8c543c..d0876697bee 100644 --- a/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloud.java +++ b/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloud.java @@ -1,6 +1,6 @@ package datawave.util.keyword; -import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; +import static datawave.marking.AccessExpressionMarkings.ACCESS; import java.util.ArrayList; import java.util.Collection; diff --git a/warehouse/query-core/src/main/java/datawave/core/iterators/FieldIndexCountingIteratorPerVisibility.java b/warehouse/query-core/src/main/java/datawave/core/iterators/FieldIndexCountingIteratorPerVisibility.java index 22cbd2f5f58..b6bcfda1961 100644 --- a/warehouse/query-core/src/main/java/datawave/core/iterators/FieldIndexCountingIteratorPerVisibility.java +++ b/warehouse/query-core/src/main/java/datawave/core/iterators/FieldIndexCountingIteratorPerVisibility.java @@ -34,8 +34,8 @@ import com.google.common.collect.Sets; -import datawave.core.common.util.AccessExpressionUtil; import datawave.iterators.IteratorSettingHelper; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.query.Constants; import datawave.util.CompositeTimestamp; diff --git a/warehouse/query-core/src/main/java/datawave/core/iterators/GlobalIndexDateSummaryIterator.java b/warehouse/query-core/src/main/java/datawave/core/iterators/GlobalIndexDateSummaryIterator.java index 898293dcc71..2cda3b801e8 100644 --- a/warehouse/query-core/src/main/java/datawave/core/iterators/GlobalIndexDateSummaryIterator.java +++ b/warehouse/query-core/src/main/java/datawave/core/iterators/GlobalIndexDateSummaryIterator.java @@ -26,9 +26,9 @@ import com.google.common.collect.Sets; import com.google.protobuf.InvalidProtocolBufferException; -import datawave.core.common.util.AccessExpressionUtil; import datawave.ingest.protobuf.Uid; import datawave.ingest.protobuf.Uid.List.Builder; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.query.Constants; diff --git a/warehouse/query-core/src/main/java/datawave/core/iterators/ResultCountingIterator.java b/warehouse/query-core/src/main/java/datawave/core/iterators/ResultCountingIterator.java index 3a92a617593..ffa894aff88 100644 --- a/warehouse/query-core/src/main/java/datawave/core/iterators/ResultCountingIterator.java +++ b/warehouse/query-core/src/main/java/datawave/core/iterators/ResultCountingIterator.java @@ -32,7 +32,7 @@ import com.google.common.cache.CacheBuilder; import com.google.common.collect.Sets; -import datawave.core.common.util.AccessExpressionUtil; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; /** diff --git a/warehouse/query-core/src/main/java/datawave/query/attributes/Attribute.java b/warehouse/query-core/src/main/java/datawave/query/attributes/Attribute.java index a9d5f3a258a..8c4d33729f0 100644 --- a/warehouse/query-core/src/main/java/datawave/query/attributes/Attribute.java +++ b/warehouse/query-core/src/main/java/datawave/query/attributes/Attribute.java @@ -27,7 +27,7 @@ import datawave.core.cache.CaffeineClassCache; import datawave.core.cache.ClassCache; -import datawave.core.common.util.AccessExpressionUtil; +import datawave.marking.AccessExpressionUtil; import datawave.query.Constants; import datawave.query.jexl.DatawaveJexlContext; diff --git a/warehouse/query-core/src/main/java/datawave/query/attributes/AttributeBag.java b/warehouse/query-core/src/main/java/datawave/query/attributes/AttributeBag.java index 2a9b1846e89..9d69e02cd79 100644 --- a/warehouse/query-core/src/main/java/datawave/query/attributes/AttributeBag.java +++ b/warehouse/query-core/src/main/java/datawave/query/attributes/AttributeBag.java @@ -10,7 +10,7 @@ import org.apache.commons.lang.mutable.MutableLong; import org.apache.log4j.Logger; -import datawave.core.common.util.AccessExpressionUtil; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.MarkingFunctions.Exception; diff --git a/warehouse/query-core/src/main/java/datawave/query/attributes/Document.java b/warehouse/query-core/src/main/java/datawave/query/attributes/Document.java index ee954e63929..b28d57538af 100644 --- a/warehouse/query-core/src/main/java/datawave/query/attributes/Document.java +++ b/warehouse/query-core/src/main/java/datawave/query/attributes/Document.java @@ -33,7 +33,7 @@ import com.google.common.collect.Iterators; import com.google.common.collect.Sets; -import datawave.core.common.util.AccessExpressionUtil; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.query.Constants; diff --git a/warehouse/query-core/src/main/java/datawave/query/common/grouping/GroupingUtils.java b/warehouse/query-core/src/main/java/datawave/query/common/grouping/GroupingUtils.java index fd0692a9772..f63cd8dcbd6 100644 --- a/warehouse/query-core/src/main/java/datawave/query/common/grouping/GroupingUtils.java +++ b/warehouse/query-core/src/main/java/datawave/query/common/grouping/GroupingUtils.java @@ -16,8 +16,8 @@ import com.google.common.base.Preconditions; -import datawave.core.common.util.AccessExpressionUtil; import datawave.data.type.NumberType; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.query.attributes.Document; import datawave.query.attributes.TypeAttribute; diff --git a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java index 934c82e8275..dd5f002eb33 100644 --- a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java +++ b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java @@ -30,8 +30,8 @@ import com.google.common.collect.Multimap; import com.google.protobuf.InvalidProtocolBufferException; -import datawave.core.common.util.AccessExpressionUtil; import datawave.ingest.protobuf.Uid; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.query.Constants; diff --git a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryTransformer.java index 050521bece3..74d951bdc17 100644 --- a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryTransformer.java @@ -9,10 +9,10 @@ import com.google.common.base.Preconditions; -import datawave.core.common.util.AccessExpressionUtil; import datawave.core.query.cachedresults.CacheableLogic; import datawave.core.query.logic.BaseQueryLogic; import datawave.core.query.logic.BaseQueryLogicTransformer; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.microservice.query.Query; diff --git a/warehouse/query-core/src/main/java/datawave/query/discovery/TermInfoAggregation.java b/warehouse/query-core/src/main/java/datawave/query/discovery/TermInfoAggregation.java index ae261366891..6a5323103aa 100644 --- a/warehouse/query-core/src/main/java/datawave/query/discovery/TermInfoAggregation.java +++ b/warehouse/query-core/src/main/java/datawave/query/discovery/TermInfoAggregation.java @@ -16,7 +16,7 @@ import com.google.common.base.Function; import com.google.common.collect.Sets; -import datawave.core.common.util.AccessExpressionUtil; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; public class TermInfoAggregation implements Function,DiscoveredThing> { diff --git a/warehouse/query-core/src/main/java/datawave/query/iterators/FieldIndexCountingIterator.java b/warehouse/query-core/src/main/java/datawave/query/iterators/FieldIndexCountingIterator.java index b3d640535a4..bf22f948828 100644 --- a/warehouse/query-core/src/main/java/datawave/query/iterators/FieldIndexCountingIterator.java +++ b/warehouse/query-core/src/main/java/datawave/query/iterators/FieldIndexCountingIterator.java @@ -30,9 +30,9 @@ import org.apache.hadoop.io.Text; import org.apache.log4j.Logger; -import datawave.core.common.util.AccessExpressionUtil; import datawave.core.common.util.TypeFilter; import datawave.iterators.IteratorSettingHelper; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.query.Constants; import datawave.util.CompositeTimestamp; diff --git a/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java b/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java index 498ad22106b..7f32cd69d33 100644 --- a/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java +++ b/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java @@ -44,8 +44,8 @@ import com.google.common.collect.Maps; -import datawave.core.common.util.AccessExpressionUtil; import datawave.core.iterators.DatawaveFieldIndexListIteratorJexl; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.MarkingFunctionsFactory; import datawave.query.attributes.Attribute; diff --git a/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java b/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java index 3434dbc787d..4cf3344e7ae 100644 --- a/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java +++ b/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java @@ -22,10 +22,10 @@ import com.google.common.collect.Multimap; import com.google.common.collect.Multimaps; -import datawave.core.common.util.AccessExpressionUtil; import datawave.data.type.OneToManyNormalizerType; import datawave.data.type.Type; import datawave.ingest.data.config.ingest.CompositeIngest; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.MarkingFunctions.Exception; import datawave.query.attributes.Attribute; diff --git a/warehouse/query-core/src/main/java/datawave/query/table/parser/EventKeyValueFactory.java b/warehouse/query-core/src/main/java/datawave/query/table/parser/EventKeyValueFactory.java index bc94602e268..88b012323bb 100644 --- a/warehouse/query-core/src/main/java/datawave/query/table/parser/EventKeyValueFactory.java +++ b/warehouse/query-core/src/main/java/datawave/query/table/parser/EventKeyValueFactory.java @@ -5,7 +5,7 @@ import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.ColumnVisibility; -import datawave.core.common.util.AccessExpressionUtil; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.query.Constants; diff --git a/warehouse/query-core/src/main/java/datawave/query/table/parser/TermFrequencyKeyValueFactory.java b/warehouse/query-core/src/main/java/datawave/query/table/parser/TermFrequencyKeyValueFactory.java index 895c2eebd98..615ef7b56e9 100644 --- a/warehouse/query-core/src/main/java/datawave/query/table/parser/TermFrequencyKeyValueFactory.java +++ b/warehouse/query-core/src/main/java/datawave/query/table/parser/TermFrequencyKeyValueFactory.java @@ -9,8 +9,8 @@ import com.google.protobuf.InvalidProtocolBufferException; -import datawave.core.common.util.AccessExpressionUtil; import datawave.ingest.protobuf.TermWeight; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.query.table.parser.EventKeyValueFactory.EventKeyValue; diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountAggregatingIterator.java b/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountAggregatingIterator.java index 4ba85309b06..a20b7d62baf 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountAggregatingIterator.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountAggregatingIterator.java @@ -26,7 +26,7 @@ import com.esotericsoftware.kryo.io.Input; import com.google.common.collect.Maps; -import datawave.core.common.util.AccessExpressionUtil; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.webservice.query.result.event.DefaultEvent; diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountResultPostprocessor.java b/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountResultPostprocessor.java index a502128fd75..683cad0604b 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountResultPostprocessor.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountResultPostprocessor.java @@ -12,8 +12,8 @@ import com.google.common.collect.Sets; -import datawave.core.common.util.AccessExpressionUtil; import datawave.core.query.logic.ResultPostprocessor; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.webservice.query.result.event.EventBase; import datawave.webservice.query.result.event.FieldBase; diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java index 3e0b3487505..1464c3f3d8a 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java @@ -30,10 +30,10 @@ import org.apache.hadoop.io.Text; import org.apache.log4j.Logger; -import datawave.core.common.util.AccessExpressionUtil; import datawave.core.query.configuration.GenericQueryConfiguration; import datawave.core.query.logic.QueryLogicTransformer; import datawave.data.type.Type; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.microservice.query.Query; import datawave.query.Constants; diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java index 2c53e873933..c978a3edfa6 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java @@ -11,12 +11,12 @@ import com.google.common.base.Preconditions; -import datawave.core.common.util.AccessExpressionUtil; import datawave.core.query.exception.EmptyObjectException; import datawave.core.query.logic.BaseQueryLogic; import datawave.core.query.logic.Flushable; import datawave.core.query.logic.WritesQueryMetrics; import datawave.core.query.logic.WritesResultCardinalities; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.microservice.query.Query; diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformerSupport.java b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformerSupport.java index 8d2b9133cd4..bbed63d923f 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformerSupport.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformerSupport.java @@ -1,5 +1,7 @@ package datawave.query.transformer; +import static datawave.marking.AccessExpressionMarkings.ACCESS; + import java.io.File; import java.io.IOException; import java.text.SimpleDateFormat; @@ -15,7 +17,6 @@ import java.util.Map.Entry; import java.util.Set; -import org.apache.accumulo.access.impl.AccessExpressionImpl; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.ColumnVisibility; @@ -25,12 +26,12 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import datawave.core.common.util.AccessExpressionUtil; import datawave.core.query.exception.EmptyObjectException; import datawave.core.query.logic.BaseQueryLogic; import datawave.core.query.logic.WritesQueryMetrics; import datawave.core.query.logic.WritesResultCardinalities; import datawave.marking.AccessExpressionMarkings; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.microservice.query.Query; @@ -72,7 +73,7 @@ public abstract class DocumentTransformerSupport extends EventQueryTransfor protected Boolean reducedResponse; private static final Logger log = Logger.getLogger(DocumentTransformerSupport.class); - private static final Markings EMPTY_MARKINGS = AccessExpressionMarkings.builder().accessExpression(AccessExpressionImpl.EMPTY).build(); + private static final Markings EMPTY_MARKINGS = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression("")).build(); private long sourceCount = 0; private long nextCount = 0; diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformer.java index 7070534f10e..c9bc4d570da 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformer.java @@ -11,12 +11,12 @@ import com.google.protobuf.InvalidProtocolBufferException; -import datawave.core.common.util.AccessExpressionUtil; import datawave.core.query.cachedresults.CacheableLogic; import datawave.edge.model.EdgeModelFields; import datawave.edge.util.EdgeKey; import datawave.edge.util.EdgeValue; import datawave.edge.util.EdgeValueHelper; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.microservice.query.Query; diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformerSupport.java b/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformerSupport.java index 40cfaef17bf..81704053f57 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformerSupport.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformerSupport.java @@ -14,11 +14,11 @@ import com.google.protobuf.InvalidProtocolBufferException; -import datawave.core.common.util.AccessExpressionUtil; import datawave.core.query.cachedresults.CacheableLogic; import datawave.core.query.logic.BaseQueryLogicTransformer; import datawave.edge.model.EdgeModelFields; import datawave.edge.util.EdgeValue; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.microservice.query.Query; diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java index 955cf9ca081..93f74e416d3 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java @@ -14,9 +14,9 @@ import com.esotericsoftware.kryo.io.Input; -import datawave.core.common.util.AccessExpressionUtil; import datawave.core.query.cachedresults.CacheableLogic; import datawave.core.query.logic.BaseQueryLogic; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.MarkingFunctions.Exception; import datawave.marking.Markings; diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformerSupport.java b/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformerSupport.java index f80fcbe10f5..353ce22d8bc 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformerSupport.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformerSupport.java @@ -18,10 +18,10 @@ import com.esotericsoftware.kryo.Kryo; import com.google.common.collect.Lists; -import datawave.core.common.util.AccessExpressionUtil; import datawave.core.query.cachedresults.CacheableLogic; import datawave.core.query.logic.BaseQueryLogic; import datawave.core.query.logic.BaseQueryLogicTransformer; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.microservice.query.Query; diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java index 7b543f158f4..0ac3828c3e5 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java @@ -16,10 +16,10 @@ import com.google.common.base.Preconditions; -import datawave.core.common.util.AccessExpressionUtil; import datawave.core.query.exception.EmptyObjectException; import datawave.core.query.logic.BaseQueryLogic; import datawave.data.type.StringType; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.microservice.query.Query; diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/FieldIndexCountQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/FieldIndexCountQueryTransformer.java index 64813fc7e8f..202cba48e0d 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/FieldIndexCountQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/FieldIndexCountQueryTransformer.java @@ -11,11 +11,11 @@ import org.apache.hadoop.io.Text; import org.apache.log4j.Logger; -import datawave.core.common.util.AccessExpressionUtil; import datawave.core.query.cachedresults.CacheableLogic; import datawave.core.query.logic.BaseQueryLogic; import datawave.core.query.logic.BaseQueryLogicTransformer; import datawave.data.hash.UID; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.microservice.query.Query; diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/GroupingDocumentTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/GroupingDocumentTransformer.java index 9aae433f36b..22f00a339ce 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/GroupingDocumentTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/GroupingDocumentTransformer.java @@ -21,9 +21,9 @@ import com.google.common.collect.Sets; import com.google.common.collect.TreeMultimap; -import datawave.core.common.util.AccessExpressionUtil; import datawave.core.query.logic.BaseQueryLogic; import datawave.marking.AccessExpressionMarkings; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.microservice.query.Query; diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/ShardIndexQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/ShardIndexQueryTransformer.java index 07d99020b04..bc0dcecd461 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/ShardIndexQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/ShardIndexQueryTransformer.java @@ -14,11 +14,11 @@ import com.google.protobuf.InvalidProtocolBufferException; -import datawave.core.common.util.AccessExpressionUtil; import datawave.core.query.cachedresults.CacheableLogic; import datawave.core.query.logic.BaseQueryLogic; import datawave.core.query.logic.BaseQueryLogicTransformer; import datawave.ingest.protobuf.Uid; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.microservice.query.Query; diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/ShardQueryCountTableTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/ShardQueryCountTableTransformer.java index 186d81d8f78..06e24971ace 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/ShardQueryCountTableTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/ShardQueryCountTableTransformer.java @@ -9,9 +9,9 @@ import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.log4j.Logger; -import datawave.core.common.util.AccessExpressionUtil; import datawave.core.query.cachedresults.CacheableLogic; import datawave.core.query.logic.BaseQueryLogicTransformer; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.microservice.query.Query; diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/TagCloudTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/TagCloudTransformer.java index 72414f6115a..5b72686f8f1 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/TagCloudTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/TagCloudTransformer.java @@ -13,8 +13,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import datawave.core.common.util.AccessExpressionUtil; import datawave.core.query.logic.BaseQueryLogicTransformer; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.microservice.query.Query; import datawave.query.tables.keyword.KeywordQueryState; diff --git a/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleField.java b/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleField.java index 6baec2b3eb9..47bf5283b0d 100644 --- a/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleField.java +++ b/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleField.java @@ -1,6 +1,6 @@ package datawave.webservice.query.result.event; -import static org.apache.accumulo.access.examples.ParseExamples.ACCESS; +import static datawave.marking.AccessExpressionMarkings.ACCESS; import java.io.IOException; import java.io.Serializable; diff --git a/warehouse/query-core/src/test/java/datawave/query/common/grouping/DocumentGrouperTest.java b/warehouse/query-core/src/test/java/datawave/query/common/grouping/DocumentGrouperTest.java index e27697d3a7b..2526531d200 100644 --- a/warehouse/query-core/src/test/java/datawave/query/common/grouping/DocumentGrouperTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/common/grouping/DocumentGrouperTest.java @@ -1,5 +1,7 @@ package datawave.query.common.grouping; +import static datawave.marking.AccessExpressionMarkings.ACCESS; + import java.math.BigDecimal; import java.util.AbstractMap; import java.util.ArrayList; @@ -9,9 +11,7 @@ import java.util.Map; import java.util.Set; -import org.apache.accumulo.access.Access; import org.apache.accumulo.access.AccessExpression; -import org.apache.accumulo.access.examples.ParseExamples; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.security.ColumnVisibility; import org.assertj.core.util.Sets; @@ -41,7 +41,6 @@ public class DocumentGrouperTest { private static final ColumnVisibility COLVIS_I = new ColumnVisibility("I"); private static final ColumnVisibility COLVIS_ALL_E_I = new ColumnVisibility("ALL&E&I"); - private static final Access ACCESS = ParseExamples.ACCESS; private static final AccessExpression AE_ALL = ACCESS.newExpression("ALL"); private static final AccessExpression AE_E = ACCESS.newExpression("E"); private static final AccessExpression AE_I = ACCESS.newExpression("I"); diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/keyword/TagCloudTestUtil.java b/warehouse/query-core/src/test/java/datawave/query/tables/keyword/TagCloudTestUtil.java index 5b1aa9732d4..a2e69d3d945 100644 --- a/warehouse/query-core/src/test/java/datawave/query/tables/keyword/TagCloudTestUtil.java +++ b/warehouse/query-core/src/test/java/datawave/query/tables/keyword/TagCloudTestUtil.java @@ -1,12 +1,11 @@ package datawave.query.tables.keyword; +import static datawave.marking.AccessExpressionMarkings.ACCESS; + import java.util.ArrayList; import java.util.List; import java.util.Map; -import org.apache.accumulo.access.Access; -import org.apache.accumulo.access.examples.ParseExamples; - import datawave.marking.AccessExpressionMarkings; import datawave.marking.Markings; import datawave.query.tables.keyword.transform.KeywordResultsTransformer; @@ -17,7 +16,7 @@ * Utility for producing TagClouds and supporting objects for tests */ public class TagCloudTestUtil { - private static final Access ACCESS = ParseExamples.ACCESS; + private static final Markings ALL_MARKINGS = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression("ALL")).build(); public static final String SOPRANO_SOURCE = "20130101_0/test/-1kfeoq.-80b5fs.r0262j"; diff --git a/web-services/metrics/src/main/java/datawave/query/metrics/ShardTableQueryMetricHandler.java b/web-services/metrics/src/main/java/datawave/query/metrics/ShardTableQueryMetricHandler.java index d4b6ec5e4b1..0fea43042fd 100644 --- a/web-services/metrics/src/main/java/datawave/query/metrics/ShardTableQueryMetricHandler.java +++ b/web-services/metrics/src/main/java/datawave/query/metrics/ShardTableQueryMetricHandler.java @@ -60,7 +60,6 @@ import datawave.core.common.connection.AccumuloConnectionFactory; import datawave.core.common.connection.AccumuloConnectionFactory.Priority; import datawave.core.common.logging.ThreadConfigurableLogger; -import datawave.core.common.util.AccessExpressionUtil; import datawave.core.query.cache.ResultsPage; import datawave.core.query.logic.QueryLogic; import datawave.core.query.logic.QueryLogicFactory; @@ -76,6 +75,7 @@ import datawave.ingest.mapreduce.job.BulkIngestKey; import datawave.ingest.mapreduce.job.writer.LiveContextWriter; import datawave.ingest.table.config.TableConfigHelper; +import datawave.marking.AccessExpressionUtil; import datawave.marking.Markings; import datawave.microservice.query.Query; import datawave.microservice.query.QueryImpl; From 6eb53b1bc6365b71577a3bba1e169e4d606affa6 Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Thu, 2 Apr 2026 13:24:06 -0400 Subject: [PATCH 16/63] update SNAPSHOTs --- core/metrics-reporter/pom.xml | 2 +- microservices/services/file-provider/pom.xml | 2 +- microservices/services/hazelcast/pom.xml | 2 +- microservices/services/mapreduce-query/pom.xml | 2 +- microservices/services/modification/pom.xml | 2 +- microservices/services/query-executor/api/pom.xml | 2 +- microservices/services/query-executor/pom.xml | 2 +- microservices/services/query-executor/service/pom.xml | 2 +- microservices/services/query-metric/pom.xml | 2 +- microservices/services/query/pom.xml | 2 +- microservices/services/query/service/pom.xml | 2 +- microservices/starters/cached-results/pom.xml | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/core/metrics-reporter/pom.xml b/core/metrics-reporter/pom.xml index 82bc6110ee0..e5486ef3785 100644 --- a/core/metrics-reporter/pom.xml +++ b/core/metrics-reporter/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-parent - 4.0.10 + 4.0.11-SNAPSHOT ../../microservices/microservice-parent/pom.xml metrics-reporter diff --git a/microservices/services/file-provider/pom.xml b/microservices/services/file-provider/pom.xml index 71c04ac6204..eef65e751a3 100644 --- a/microservices/services/file-provider/pom.xml +++ b/microservices/services/file-provider/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-parent - 4.0.10 + 4.0.11-SNAPSHOT ../../microservice-parent/pom.xml file-provider-service-parent diff --git a/microservices/services/hazelcast/pom.xml b/microservices/services/hazelcast/pom.xml index f470546bb2a..2836e95153b 100644 --- a/microservices/services/hazelcast/pom.xml +++ b/microservices/services/hazelcast/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-parent - 4.0.10 + 4.0.11-SNAPSHOT ../../microservice-parent/pom.xml hazelcast-service-parent diff --git a/microservices/services/mapreduce-query/pom.xml b/microservices/services/mapreduce-query/pom.xml index 2905b41643a..b121b57353c 100644 --- a/microservices/services/mapreduce-query/pom.xml +++ b/microservices/services/mapreduce-query/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-parent - 4.0.10 + 4.0.11-SNAPSHOT ../../microservice-parent/pom.xml mapreduce-query-service-parent diff --git a/microservices/services/modification/pom.xml b/microservices/services/modification/pom.xml index a7613e4bb08..92d0e3f87f4 100644 --- a/microservices/services/modification/pom.xml +++ b/microservices/services/modification/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-parent - 4.0.10 + 4.0.11-SNAPSHOT ../../microservice-parent/pom.xml modification-service-parent diff --git a/microservices/services/query-executor/api/pom.xml b/microservices/services/query-executor/api/pom.xml index 3524463b8f5..473f0cd7f91 100644 --- a/microservices/services/query-executor/api/pom.xml +++ b/microservices/services/query-executor/api/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-parent - 4.0.10 + 4.0.11-SNAPSHOT ../../../microservice-parent/pom.xml query-executor-api diff --git a/microservices/services/query-executor/pom.xml b/microservices/services/query-executor/pom.xml index b2735ce95b6..9368cbd73dc 100644 --- a/microservices/services/query-executor/pom.xml +++ b/microservices/services/query-executor/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-parent - 4.0.10 + 4.0.11-SNAPSHOT ../../microservice-parent/pom.xml query-executor-service-parent diff --git a/microservices/services/query-executor/service/pom.xml b/microservices/services/query-executor/service/pom.xml index d1cca7fd33d..28bc581f9ea 100644 --- a/microservices/services/query-executor/service/pom.xml +++ b/microservices/services/query-executor/service/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-service-parent - 5.0.11 + 5.0.12-SNAPSHOT ../../../microservice-service-parent/pom.xml query-executor-service diff --git a/microservices/services/query-metric/pom.xml b/microservices/services/query-metric/pom.xml index 7cc9d2d3154..e5700e00c6e 100644 --- a/microservices/services/query-metric/pom.xml +++ b/microservices/services/query-metric/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-parent - 4.0.10 + 4.0.11-SNAPSHOT ../../microservice-parent/pom.xml query-metric-parent diff --git a/microservices/services/query/pom.xml b/microservices/services/query/pom.xml index 7adb505614d..8526739bc95 100644 --- a/microservices/services/query/pom.xml +++ b/microservices/services/query/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-parent - 4.0.10 + 4.0.11-SNAPSHOT ../../microservice-parent/pom.xml query-service-parent diff --git a/microservices/services/query/service/pom.xml b/microservices/services/query/service/pom.xml index c181e33b6c6..4b37ba0e793 100644 --- a/microservices/services/query/service/pom.xml +++ b/microservices/services/query/service/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-service-parent - 5.0.11 + 5.0.12-SNAPSHOT ../../../microservice-service-parent/pom.xml query-service diff --git a/microservices/starters/cached-results/pom.xml b/microservices/starters/cached-results/pom.xml index f06f16b0190..3db9435e9bc 100644 --- a/microservices/starters/cached-results/pom.xml +++ b/microservices/starters/cached-results/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-service-parent - 5.0.11 + 5.0.12-SNAPSHOT ../../microservice-service-parent/pom.xml spring-boot-starter-datawave-cached-results From cfcc027d6b878c7282d02739b35ec1820a3c70c0 Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Mon, 6 Apr 2026 16:42:34 -0400 Subject: [PATCH 17/63] add back needed methods --- .../marking/ColumnVisibilityCache.java | 22 +++++++++ .../marking/FlattenedVisibilityCache.java | 41 ++++++++++++++++ .../datawave/marking/MarkingFunctions.java | 49 ++++++++++++++++++- 3 files changed, 111 insertions(+), 1 deletion(-) create mode 100644 core/utils/accumulo-utils/src/main/java/datawave/marking/ColumnVisibilityCache.java create mode 100644 core/utils/accumulo-utils/src/main/java/datawave/marking/FlattenedVisibilityCache.java diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/ColumnVisibilityCache.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/ColumnVisibilityCache.java new file mode 100644 index 00000000000..27fe70ebdfb --- /dev/null +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/ColumnVisibilityCache.java @@ -0,0 +1,22 @@ +package datawave.marking; + +import java.util.Collections; +import java.util.Map; + +import org.apache.accumulo.core.data.ByteSequence; +import org.apache.accumulo.core.security.ColumnVisibility; +import org.apache.commons.collections4.map.LRUMap; + +public class ColumnVisibilityCache { + @SuppressWarnings("unchecked") + private static Map cache = Collections.synchronizedMap(new LRUMap(256)); + + public static ColumnVisibility get(ByteSequence bytes) { + ColumnVisibility vis = cache.get(bytes); + if (vis == null) { + vis = new ColumnVisibility(bytes.toArray()); + cache.put(bytes, vis); + } + return vis; + } +} diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/FlattenedVisibilityCache.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/FlattenedVisibilityCache.java new file mode 100644 index 00000000000..85e1697cc26 --- /dev/null +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/FlattenedVisibilityCache.java @@ -0,0 +1,41 @@ +package datawave.marking; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import org.apache.accumulo.core.data.ByteSequence; +import org.apache.accumulo.core.security.ColumnVisibility; + +/** + * This is a cache that can be used per process to save flattened visibility calculations. + * + */ +public class FlattenedVisibilityCache { + private static Map flattenedVisCache = Collections.synchronizedMap(new HashMap<>()); + + /** + * Create a flattened visibility, using the cache if possible + * + * @param vis + * the visibility to flatten + * @return the flattened visibility + */ + public static byte[] flatten(ColumnVisibility vis) { + byte[] visBytes = flattenedVisCache.get(vis); + if (visBytes == null) { + visBytes = vis.flatten(); + flattenedVisCache.put(vis, visBytes); + } + return visBytes; + } + + public static byte[] flatten(ByteSequence bytes) { + return flatten(ColumnVisibilityCache.get(bytes)); + } + + public static boolean equals(ColumnVisibility left, ColumnVisibility right) { + return Arrays.equals(flatten(left), flatten(right)); + } +} diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java index 7142f6ed2d6..3659db33d49 100644 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java @@ -2,6 +2,7 @@ import static datawave.marking.AccessExpressionMarkings.ACCESS; +import java.nio.charset.StandardCharsets; import java.util.Collection; import java.util.HashSet; import java.util.Set; @@ -9,6 +10,8 @@ import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.access.ParsedAccessExpression; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.core.security.ColumnVisibility; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.support.ClassPathXmlApplicationContext; @@ -22,8 +25,21 @@ public interface MarkingFunctions> { + ColumnVisibility combineVisibilities(Collection visibilities) throws MarkingFunctions.Exception; + T combine(Collection markings) throws MarkingFunctions.Exception; + ColumnVisibility translateToColumnVisibility(Markings markings) throws MarkingFunctions.Exception; + + Markings translateFromColumnVisibility(ColumnVisibility columnVisibility) throws MarkingFunctions.Exception; + + Markings translateFromColumnVisibilityForAuths(ColumnVisibility columnVisibility, Collection authorizations) + throws MarkingFunctions.Exception; + + Markings translateFromColumnVisibilityForAuths(ColumnVisibility columnVisibility, Authorizations authorizations) throws MarkingFunctions.Exception; + + byte[] flatten(ColumnVisibility vis); + class Exception extends java.lang.Exception { public Exception() { @@ -51,6 +67,11 @@ class Default implements MarkingFunctions { private static final AccessExpressionMarkings EMPTY_MARKINGS = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression("")).build(); + @Override + public ColumnVisibility combineVisibilities(Collection visibilities) { + throw new UnsupportedOperationException(); + } + @Override public AccessExpressionMarkings combine(Collection markings) { @@ -78,9 +99,35 @@ public AccessExpressionMarkings combine(Collection mar // @formatter:on // normalize the Parsed Expression and then return a copy without the parse tree - // FIXME: Using a beta of the accumulo-access API so temporarily using the normalize function from the example code return AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression(AccessExpressionUtil.normalize(expression).expression)).build(); } + + @Override + public ColumnVisibility translateToColumnVisibility(Markings markings) { + AccessExpression accessExpression = AccessExpressionUtil.toAccessExpression(markings); + return AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.normalize(accessExpression)); + } + + @Override + public Markings translateFromColumnVisibility(ColumnVisibility expression) { + AccessExpression accessExpression = ACCESS.newExpression(new String(expression.getExpression(), StandardCharsets.UTF_8)); + return AccessExpressionMarkings.builder().accessExpression(AccessExpressionUtil.normalize(accessExpression)).build(); + } + + @Override + public Markings translateFromColumnVisibilityForAuths(ColumnVisibility columnVisibility, Collection authorizations) { + return translateFromColumnVisibility(columnVisibility); + } + + @Override + public Markings translateFromColumnVisibilityForAuths(ColumnVisibility columnVisibility, Authorizations authorizations) { + return translateFromColumnVisibility(columnVisibility); + } + + @Override + public byte[] flatten(ColumnVisibility vis) { + return FlattenedVisibilityCache.flatten(vis); + } } /** From c0f60aa44193ac23d61191bab126b670aba27e94 Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Tue, 7 Apr 2026 13:45:43 -0400 Subject: [PATCH 18/63] tag cloud fixes --- .../java/datawave/util/keyword/DefaultTagCloudUtils.java | 5 ++--- .../src/main/java/datawave/util/keyword/TagCloud.java | 9 ++++++--- .../main/java/datawave/util/keyword/TagCloudUtils.java | 6 +++--- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/warehouse/keyword-common/src/main/java/datawave/util/keyword/DefaultTagCloudUtils.java b/warehouse/keyword-common/src/main/java/datawave/util/keyword/DefaultTagCloudUtils.java index 417643cab3c..901a12a4519 100644 --- a/warehouse/keyword-common/src/main/java/datawave/util/keyword/DefaultTagCloudUtils.java +++ b/warehouse/keyword-common/src/main/java/datawave/util/keyword/DefaultTagCloudUtils.java @@ -11,7 +11,6 @@ import java.util.Set; import java.util.stream.Collectors; -import org.apache.accumulo.access.AccessExpression; import org.apache.commons.lang3.StringUtils; import datawave.marking.AccessExpressionUtil; @@ -27,14 +26,14 @@ public class DefaultTagCloudUtils implements TagCloudUtils, Serializable { private static final String MULTI_VALUE_SEPARATOR = ","; @Override - public AccessExpression generateCombinedVisibility(Set visibilities) { + public Markings generateCombinedVisibility(Set visibilities) { if (visibilities == null || visibilities.isEmpty()) { return null; } Set> markings = visibilities.stream().map(v -> AccessExpressionUtil.toAccessExpressionMarkings(ACCESS.newExpression(v))) .collect(Collectors.toSet()); try { - return AccessExpressionUtil.toAccessExpression(AccessExpressionUtil.combine(new MarkingFunctions.Default(), markings)); + return AccessExpressionUtil.combine(new MarkingFunctions.Default(), markings); } catch (MarkingFunctions.Exception e) { throw new IllegalArgumentException("Failed to combine visibilities", e); } diff --git a/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloud.java b/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloud.java index d0876697bee..fe563a807e1 100644 --- a/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloud.java +++ b/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloud.java @@ -18,6 +18,9 @@ import com.google.gson.Gson; +import datawave.marking.AccessExpressionMarkings; +import datawave.marking.Markings; + /** A tag cloud - a collection of tags that have a keyword, score, frequency and list of sources from which they originated */ public class TagCloud { @@ -42,9 +45,9 @@ public class TagCloud { * @param results * the entries that belong in the tag cloud. */ - protected TagCloud(Map metadata, AccessExpression visibility, SortedSet results) { + protected TagCloud(Map metadata, Markings visibility, SortedSet results) { this.metadata = metadata; - this.visibility = (visibility != null) ? visibility.getExpression() : ""; + this.visibility = (visibility != null) ? ((AccessExpressionMarkings) visibility).getMarkings().getExpression() : ""; this.results = results; } @@ -221,7 +224,7 @@ public List build() { final SortedSet results = new TreeSet<>(comparator); results.addAll(e.getValue()); String partition = getPartition(e.getKey()); - AccessExpression visibility = utils.generateCombinedVisibility(visibilities.get(partition)); + Markings visibility = utils.generateCombinedVisibility(visibilities.get(partition)); Map tagCloudMetadata = utils.generateCombinedMetadata(this.metadata.get(partition)); tagClouds.add(new TagCloud(tagCloudMetadata, visibility, results)); } diff --git a/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloudUtils.java b/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloudUtils.java index 23fcb5d4c23..dc837b0ee52 100644 --- a/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloudUtils.java +++ b/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloudUtils.java @@ -5,7 +5,7 @@ import java.util.Map; import java.util.Set; -import org.apache.accumulo.access.AccessExpression; +import datawave.marking.Markings; /** * Interface for pluggable utilities for generating tag clouds, includes mechanisms to partition keywords into separate tag clouds, combine or merge visibility @@ -17,9 +17,9 @@ public interface TagCloudUtils { * * @param visibilities * a set of valid visibility strings. - * @return a combined {@link AccessExpression}, or {@code null} if there are no visibilities + * @return a combined {@link Markings}, or {@code null} if there are no visibilities */ - AccessExpression generateCombinedVisibility(Set visibilities); + Markings generateCombinedVisibility(Set visibilities); /** * Aggregate metadata from multiple sources, combining multi-valued entries into a flattened string From 05227732175f9e41424bca7a09ae2e04fb30e18d Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Wed, 8 Apr 2026 09:49:15 -0400 Subject: [PATCH 19/63] fix FieldBase and FieldCardinalityBase column visibility --- .../datawave/webservice/query/result/event/FieldBase.java | 8 ++++++++ .../query/result/event/FieldCardinalityBase.java | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/web-services/client/src/main/java/datawave/webservice/query/result/event/FieldBase.java b/web-services/client/src/main/java/datawave/webservice/query/result/event/FieldBase.java index a3119cd2530..30c7b8069d6 100644 --- a/web-services/client/src/main/java/datawave/webservice/query/result/event/FieldBase.java +++ b/web-services/client/src/main/java/datawave/webservice/query/result/event/FieldBase.java @@ -1,10 +1,13 @@ package datawave.webservice.query.result.event; +import java.nio.charset.StandardCharsets; + import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlSeeAlso; import org.apache.accumulo.access.AccessExpression; +import org.apache.accumulo.core.security.ColumnVisibility; import datawave.marking.Markings; import datawave.webservice.query.util.TypedValue; @@ -36,6 +39,11 @@ public abstract class FieldBase implements HasMarkings, Message { public abstract void setColumnVisibility(String columnVisibility); + public void setColumnVisibility(ColumnVisibility columnVisibility) { + String exprString = (columnVisibility == null) ? null : new String(columnVisibility.getExpression(), StandardCharsets.UTF_8); + setColumnVisibility(exprString); + } + public void setColumnVisibility(AccessExpression accessExpression) { String exprString = (accessExpression == null) ? null : accessExpression.getExpression(); setColumnVisibility(exprString); diff --git a/web-services/client/src/main/java/datawave/webservice/query/result/event/FieldCardinalityBase.java b/web-services/client/src/main/java/datawave/webservice/query/result/event/FieldCardinalityBase.java index 73bfc3125d9..cb14e3eae84 100644 --- a/web-services/client/src/main/java/datawave/webservice/query/result/event/FieldCardinalityBase.java +++ b/web-services/client/src/main/java/datawave/webservice/query/result/event/FieldCardinalityBase.java @@ -1,10 +1,13 @@ package datawave.webservice.query.result.event; +import java.nio.charset.StandardCharsets; + import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlSeeAlso; import org.apache.accumulo.access.AccessExpression; +import org.apache.accumulo.core.security.ColumnVisibility; import datawave.marking.Markings; @@ -21,6 +24,11 @@ public abstract class FieldCardinalityBase implements HasMarkings { public abstract void setColumnVisibility(String columnVisibility); + public void setColumnVisibility(ColumnVisibility columnVisibility) { + String exprString = (columnVisibility == null) ? null : new String(columnVisibility.getExpression(), StandardCharsets.UTF_8); + setColumnVisibility(exprString); + } + public void setColumnVisibility(AccessExpression accessExpression) { String exprString = (accessExpression == null) ? null : accessExpression.getExpression(); setColumnVisibility(exprString); From d13ae1e0caa17545f562d5a61f37abec9a517e2f Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Wed, 8 Apr 2026 13:53:20 -0400 Subject: [PATCH 20/63] cleanup dead marking code --- .../ingest/config/RawRecordContainerImpl.java | 19 ----- .../ingest/data/RawRecordContainer.java | 4 - .../tokenize/ExtendedContentIngestHelper.java | 84 ------------------- .../ingest/mapreduce/SimpleRawRecord.java | 17 ---- .../edge/ProtobufEdgeDeleteModeTest.java | 2 +- .../ProtobufEdgeDeletePreconditionTest.java | 2 +- .../edge/ProtobufEdgeDirectionTest.java | 2 +- .../edge/ProtobufEdgePreconditionTest.java | 2 +- .../query/result/event/SimpleField.java | 4 - .../modification/ModificationRequestBase.java | 2 +- 10 files changed, 5 insertions(+), 133 deletions(-) diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/config/RawRecordContainerImpl.java b/warehouse/ingest-core/src/main/java/datawave/ingest/config/RawRecordContainerImpl.java index b2d62853beb..968d5f4a27e 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/config/RawRecordContainerImpl.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/config/RawRecordContainerImpl.java @@ -101,25 +101,6 @@ public void setSecurityMarkings(Markings securityMarkings) { syncSecurityMarkingsToFields(); } - @Override - public void addSecurityMarking(String domain, String marking) { - // Legacy map-based operation — treat marking as an access expression string - this.securityMarkings = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression(marking)).build(); - syncSecurityMarkingsToFields(); - } - - @Override - public boolean hasSecurityMarking(String domain, String marking) { - if (securityMarkings == null) { - return false; - } - Object m = securityMarkings.getMarkings(); - if (m instanceof AccessExpression) { - return ((AccessExpression) m).getExpression().equals(marking); - } - return false; - } - protected void syncSecurityMarkingsToFields() { if (securityMarkings != null) { Object m = securityMarkings.getMarkings(); diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/RawRecordContainer.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/RawRecordContainer.java index 1e0106022ef..1628282835f 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/RawRecordContainer.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/RawRecordContainer.java @@ -21,10 +21,6 @@ public interface RawRecordContainer { void setSecurityMarkings(Markings securityMarkings); - void addSecurityMarking(String domain, String marking); - - boolean hasSecurityMarking(String domain, String marking); - UID getId(); void setId(UID id); diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/tokenize/ExtendedContentIngestHelper.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/tokenize/ExtendedContentIngestHelper.java index d44e74af1f9..b3c0dd5388d 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/tokenize/ExtendedContentIngestHelper.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/tokenize/ExtendedContentIngestHelper.java @@ -5,7 +5,6 @@ import java.util.Map; import java.util.Set; -import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.log4j.Logger; @@ -170,89 +169,6 @@ public void addMetadataFromId(RawRecordContainer event, String id) { } } - public void inheritFromRootPayload(RawRecordContainer event, Map> metadata) { - // TODO: No-op here, but need to refactor the following in upstream - // if (metadata == null) - // return; - // - // IBaseDataObject payload = (IBaseDataObject) event.getAuxData(); - // if (payload != null) { - // for (String field : helper.getInheritedPayloadFields()) { - // if (metadata.containsKey(field) && (!payload.hasParameter(field))) { - // payload.putParameter(field, metadata.get(field)); - // } - // } - // } - } - - public void addMetadataFromParms(RawRecordContainer event, Map> metadata, String id) { - - for (Map.Entry> entry : metadata.entrySet()) { - CharSequence key = entry.getKey(); - for (Object value : entry.getValue()) { - if (value == null) - continue; - - if (helper.getUuids().contains(key)) { - event.getAltIds().add(String.valueOf(value)); - } - if (this.helper.getSecurityMarkingFieldDomainMap().containsKey(key)) { - addSecurityMetadataFromParms(key, value, event); - } - } - } - - Multimap newMetadata = HashMultimap.create(); - Multimap fieldParsers = helper.getMetadataFieldParsers(); - for (String field : fieldParsers.keySet()) { - if (metadata.containsKey(field) && metadata.get(field) != null) { - for (MetadataIdParser parser : fieldParsers.get(field)) { - for (Object v : metadata.get(field)) { - if (v == null) - continue; - try { - parser.addMetadata(event, newMetadata, v.toString()); - } catch (Exception e) { - log.error("Unable to apply " + parser + " to " + field, e); - event.addError(RawDataErrorNames.FIELD_EXTRACTION_ERROR); - } - } - } - } - } - - // If the ID is a UUID (i.e., contains no date), try to get - // event metadata from the parameters - if ((id != null) && (id.length() >= UUID_LENGTH) && id.matches(UUID_PATTERN)) { - fieldParsers = helper.getMetadataFieldUuidParsers(); - for (String field : fieldParsers.keySet()) { - if (metadata.containsKey(field) && metadata.get(field) != null) { - for (MetadataIdParser parser : fieldParsers.get(field)) { - for (Object v : metadata.get(field)) { - if (v == null) - continue; - try { - parser.addMetadata(event, newMetadata, v.toString()); - } catch (Exception e) { - log.error("Unable to apply " + parser + " to " + field, e); - event.addError(RawDataErrorNames.FIELD_EXTRACTION_ERROR); - } - } - } - } - } - } - } - - protected void addSecurityMetadataFromParms(CharSequence key, Object value, RawRecordContainer event) { - // If fieldName is a security marking field (as configured by EVENT_SECURITY_MARKING_FIELD_NAMES), - // then put the marking value into this.securityMarkings, where 'key' maps to the domain for the marking - // (as configured by EVENT_SECURITY_MARKING_FIELD_DOMAINS) - if (!StringUtils.isEmpty(key.toString()) && !StringUtils.isEmpty(value.toString())) { - event.addSecurityMarking(this.helper.getSecurityMarkingFieldDomainMap().get(key.toString()), value.toString()); - } - } - /** * Override the normalize call to enable event field value normalization * diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/SimpleRawRecord.java b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/SimpleRawRecord.java index adfc53f201c..e06de51b000 100644 --- a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/SimpleRawRecord.java +++ b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/SimpleRawRecord.java @@ -58,23 +58,6 @@ public void setSecurityMarkings(Markings securityMarkings) { this.securityMarkings = securityMarkings; } - @Override - public void addSecurityMarking(String domain, String marking) { - this.securityMarkings = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression(marking)).build(); - } - - @Override - public boolean hasSecurityMarking(String domain, String marking) { - if (securityMarkings == null) { - return false; - } - Object m = securityMarkings.getMarkings(); - if (m instanceof AccessExpression) { - return ((AccessExpression) m).getExpression().equals(marking); - } - return false; - } - @Override public UID getId() { return id; diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDeleteModeTest.java b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDeleteModeTest.java index 5e96b93168e..ecceef27fb5 100644 --- a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDeleteModeTest.java +++ b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDeleteModeTest.java @@ -131,7 +131,7 @@ public void setup() { private RawRecordContainer getEvent(Configuration conf) { RawRecordContainerImpl myEvent = new RawRecordContainerImpl(); - myEvent.addSecurityMarking("columnVisibility", "PRIVATE"); + myEvent.setVisibility("PRIVATE"); myEvent.setDataType(type); myEvent.setId(UID.builder().newId()); myEvent.setConf(conf); diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDeletePreconditionTest.java b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDeletePreconditionTest.java index fbbc463eb24..7b8c2868ad4 100644 --- a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDeletePreconditionTest.java +++ b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDeletePreconditionTest.java @@ -62,7 +62,7 @@ public void setup() { private RawRecordContainer getEvent(Configuration conf) { RawRecordContainerImpl myEvent = new RawRecordContainerImpl(); - myEvent.addSecurityMarking("columnVisibility", "PRIVATE"); + myEvent.setVisibility("PRIVATE"); myEvent.setDataType(type); myEvent.setId(UID.builder().newId()); myEvent.setConf(conf); diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDirectionTest.java b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDirectionTest.java index 90fbb1a720a..aa2f79c44c4 100644 --- a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDirectionTest.java +++ b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDirectionTest.java @@ -319,7 +319,7 @@ public void testUniDirectionWithJexlCondition() { private RawRecordContainer getEvent(Type type) { RawRecordContainerImpl myEvent = new RawRecordContainerImpl(); - myEvent.addSecurityMarking("columnVisibility", "PRIVATE"); + myEvent.setVisibility("PRIVATE"); myEvent.setDataType(type); myEvent.setId(UID.builder().newId()); myEvent.setAltIds(Collections.singleton("0016dd72-0000-827d-dd4d-001b2163ba09")); diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgePreconditionTest.java b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgePreconditionTest.java index 8ff47a246f9..c1a56a8c13a 100644 --- a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgePreconditionTest.java +++ b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgePreconditionTest.java @@ -72,7 +72,7 @@ public void setup() { private RawRecordContainer getEvent(Configuration conf) { RawRecordContainerImpl myEvent = new RawRecordContainerImpl(); - myEvent.addSecurityMarking("columnVisibility", "PRIVATE"); + myEvent.setVisibility("PRIVATE"); myEvent.setDataType(type); myEvent.setId(UID.builder().newId()); myEvent.setAltIds(Collections.singleton("0016dd72-0000-827d-dd4d-001b2163ba09")); diff --git a/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleField.java b/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleField.java index 47bf5283b0d..59f17af0599 100644 --- a/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleField.java +++ b/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleField.java @@ -136,10 +136,6 @@ public void setName(String name) { this.name = name; } - private void assureMarkings() { - // no-op: markings are now Markings and should be set externally - } - @Override public Markings getMarkings() { return this.markings; diff --git a/web-services/client/src/main/java/datawave/webservice/modification/ModificationRequestBase.java b/web-services/client/src/main/java/datawave/webservice/modification/ModificationRequestBase.java index d52c9e16201..f2db7ad4e56 100644 --- a/web-services/client/src/main/java/datawave/webservice/modification/ModificationRequestBase.java +++ b/web-services/client/src/main/java/datawave/webservice/modification/ModificationRequestBase.java @@ -72,7 +72,7 @@ public static class DefaultFieldMarkingsAdapter extends XmlAdapter unmarshal(DefaultFieldMarking value) throws Exception { - HashMap fieldMarkings = new HashMap(); + HashMap fieldMarkings = new HashMap<>(); fieldMarkings.put(COLUMN_VISIBILITY, value.fieldColumnVisibility); return fieldMarkings; } From 1e23bf875cadeed3e823637d42a2d4f9cc0a0f27 Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Mon, 13 Apr 2026 15:18:06 -0400 Subject: [PATCH 21/63] fix SecurityMarking interface --- .../java/datawave/marking/ColumnVisibilitySecurityMarking.java | 2 +- .../src/main/java/datawave/marking/SecurityMarking.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/base-rest-responses/src/main/java/datawave/marking/ColumnVisibilitySecurityMarking.java b/core/base-rest-responses/src/main/java/datawave/marking/ColumnVisibilitySecurityMarking.java index b5576800956..86512b4cc0d 100644 --- a/core/base-rest-responses/src/main/java/datawave/marking/ColumnVisibilitySecurityMarking.java +++ b/core/base-rest-responses/src/main/java/datawave/marking/ColumnVisibilitySecurityMarking.java @@ -54,7 +54,7 @@ public AccessExpression toAccessExpression() { } @Override - public Markings toMarkings() { + public Markings toMarkings() { return AccessExpressionMarkings.builder().accessExpression(toAccessExpression()).build(); } diff --git a/core/base-rest-responses/src/main/java/datawave/marking/SecurityMarking.java b/core/base-rest-responses/src/main/java/datawave/marking/SecurityMarking.java index 78b77c764b3..bb7919559fc 100644 --- a/core/base-rest-responses/src/main/java/datawave/marking/SecurityMarking.java +++ b/core/base-rest-responses/src/main/java/datawave/marking/SecurityMarking.java @@ -10,7 +10,7 @@ public interface SecurityMarking extends ParameterValidator { String toAccessExpressionString(); - Markings toMarkings() throws MarkingFunctions.Exception; + Markings toMarkings() throws MarkingFunctions.Exception; void clear(); } From e9ee8e824f7f3cf180455c02372add9180188fe8 Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Fri, 17 Apr 2026 15:59:40 -0400 Subject: [PATCH 22/63] marking fixes --- .../cachedresults/CacheableQueryRowImpl.java | 6 +- .../modification/MutableMetadataHandler.java | 2 +- .../marking/AccessExpressionMarkings.java | 11 ++ .../marking/AccessExpressionUtil.java | 128 ++++++++++-------- .../datawave/marking/MarkingFunctions.java | 2 +- .../main/java/datawave/marking/Markings.java | 7 + .../FrequencyMetadataAggregator.java | 5 +- .../metadata/MetadataDescriptionsHelper.java | 4 +- .../querymetric/QueryMetricOperations.java | 3 +- .../handler/ShardTableQueryMetricHandler.java | 2 +- .../data/config/ingest/CompositeIngest.java | 8 +- .../handler/atom/AtomDataTypeHandler.java | 2 +- .../atom/AtomErrorDataTypeHandler.java | 5 +- .../edge/ProtobufEdgeDataTypeHandler.java | 2 +- .../handler/error/ErrorDataTypeHandler.java | 4 +- .../error/ErrorShardedDataTypeHandler.java | 5 +- .../handler/shard/ShardedDataTypeHandler.java | 2 +- ...eldIndexCountingIteratorPerVisibility.java | 5 +- .../GlobalIndexDateSummaryIterator.java | 6 +- .../iterators/ResultCountingIterator.java | 6 +- .../query/attributes/AttributeBag.java | 6 +- .../query/common/grouping/GroupingUtils.java | 5 +- .../query/discovery/DiscoveryIterator.java | 5 +- .../query/discovery/TermInfoAggregation.java | 5 +- .../iterators/FieldIndexCountingIterator.java | 6 +- .../query/jexl/DatawaveInterpreter.java | 6 +- .../query/predicate/ValueToAttributes.java | 6 +- .../table/parser/EventKeyValueFactory.java | 4 +- .../shard/CountAggregatingIterator.java | 5 +- .../shard/CountResultPostprocessor.java | 5 +- .../shard/FieldIndexCountQueryLogic.java | 5 +- .../transformer/EventQueryTransformer.java | 3 +- .../query/transformer/FacetedTransformer.java | 2 +- .../DefaultModificationRequest.java | 4 +- .../metrics/ShardTableQueryMetricHandler.java | 3 +- 35 files changed, 158 insertions(+), 127 deletions(-) diff --git a/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java b/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java index 0a1f789345f..43c2e08fa68 100644 --- a/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java +++ b/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java @@ -61,7 +61,7 @@ public void addColumn(String columnName, TypedValue columnTypedValue, Markings markings) { // validate the markings - AccessExpressionUtil.toAccessExpression(markings); + markings.toAccessExpression(); this.markings = markings; } @@ -301,7 +301,7 @@ public String getColumnSecurityMarkingString(Map columnMap) { String v = columnColumnVisibilityMap.get(field); String mStr = ""; if (m != null && !m.isEmpty()) { - mStr = AccessExpressionUtil.toAccessExpression(m).getExpression(); + mStr = m.toAccessExpression().getExpression(); } if (v == null) { combinedMap.put(field, mStr); diff --git a/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java b/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java index dbb003917b1..b18930ca0ff 100644 --- a/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java +++ b/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java @@ -661,7 +661,7 @@ protected void insert(MultiTableBatchWriter writer, String shardId, String datat if (null == markings || markings.isEmpty()) throw new IllegalArgumentException("No security information specified. Security markings must be supplied"); - viz = AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.toAccessExpression(markings)); + viz = AccessExpressionUtil.toColumnVisibility(markings.toAccessExpression()); } insert(writer, shardId, datatype, eventUid, viz, fieldName, fieldValue, ts, isIndexOnlyField, isIndexed, isReverseIndexed, dataTypes, false, diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java index bd34e94e51a..d0856f0f596 100644 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java @@ -2,6 +2,7 @@ import org.apache.accumulo.access.Access; import org.apache.accumulo.access.AccessExpression; +import org.apache.accumulo.core.security.ColumnVisibility; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonCreator; @@ -25,6 +26,16 @@ public AccessExpression getMarkings() { return getAccessExpression(); } + @Override + public AccessExpression toAccessExpression() { + return accessExpression; + } + + @Override + public ColumnVisibility toColumnVisibility() { + return AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.normalize(accessExpression)); + } + @Override public boolean isEmpty() { return accessExpression == null || accessExpression.getExpression().isEmpty(); diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionUtil.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionUtil.java index 3835bf72d19..7e2bd60872e 100644 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionUtil.java +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionUtil.java @@ -25,9 +25,25 @@ public final class AccessExpressionUtil { private static final AccessExpression EMPTY_EXPRESSION = ACCESS.newExpression(""); + private static final ColumnVisibility EMPTY_VISIBILITY = new ColumnVisibility(); private AccessExpressionUtil() {} + /** + * Convert an {@link AccessExpression} to a {@link ColumnVisibility}. + * + * @param ae + * the access expression + * @return the equivalent column visibility + */ + public static ColumnVisibility toColumnVisibility(AccessExpression ae) { + String expr = ae.getExpression(); + if (expr.isEmpty()) { + return EMPTY_VISIBILITY; + } + return new ColumnVisibility(expr); + } + /** * Convert a {@link ColumnVisibility} to an {@link AccessExpression}. * @@ -44,58 +60,17 @@ public static AccessExpression toAccessExpression(ColumnVisibility cv) { } /** - * Convert a {@link Markings} to an {@link AccessExpression}. - * - * @param markings - * assumed to be of type AccessExpressionMarkings - * @return the embedded access expression - */ - public static AccessExpression toAccessExpression(Markings markings) { - return (AccessExpression) markings.getMarkings(); - } - - /** - * Convert a {@link AccessExpression} to an {@link AccessExpressionMarkings}. - * - * @param expression - * The AccessExpression to embed in the wrapper class - * @return the wrapped access expression - */ - public static AccessExpressionMarkings toAccessExpressionMarkings(AccessExpression expression) { - return AccessExpressionMarkings.builder().accessExpression(expression).build(); - } - - /** - * Convert an {@link Markings} to a {@link ColumnVisibility}. - * - * @param markings - * the access expression - * @return the equivalent column visibility - */ - public static ColumnVisibility toColumnVisibility(Markings markings) { - if (markings instanceof AccessExpressionMarkings) { - return toColumnVisibility((AccessExpression) markings.getMarkings()); - } - String m = markings.toString(); - if (m.isEmpty()) { - return new ColumnVisibility(); - } - return new ColumnVisibility(m); - } - - /** - * Convert an {@link AccessExpression} to a {@link ColumnVisibility}. + * Convert a visibility String to an {@link AccessExpression}. * - * @param ae - * the access expression - * @return the equivalent column visibility + * @param visibility + * the visibility String + * @return the equivalent access expression */ - public static ColumnVisibility toColumnVisibility(AccessExpression ae) { - String expr = ae.getExpression(); - if (expr.isEmpty()) { - return new ColumnVisibility(); + public static AccessExpression toAccessExpression(String visibility) { + if (visibility == null || visibility.isEmpty()) { + return EMPTY_EXPRESSION; } - return new ColumnVisibility(expr); + return ACCESS.newExpression(visibility); } /** @@ -112,6 +87,17 @@ public static AccessExpression toAccessExpression(byte[] visibilityBytes) { return ACCESS.newExpression(new String(visibilityBytes, UTF_8)); } + /** + * Convert a {@link AccessExpression} to an {@link AccessExpressionMarkings}. + * + * @param expression + * The AccessExpression to embed in the wrapper class + * @return the wrapped access expression + */ + public static AccessExpressionMarkings toAccessExpressionMarkings(AccessExpression expression) { + return AccessExpressionMarkings.builder().accessExpression(expression).build(); + } + /** * Normalize an {@link AccessExpression} by deduplicating and sorting terms. This is the replacement for the old {@code ColumnVisibility.flatten()} * behavior. @@ -156,8 +142,9 @@ public static AccessExpression emptyExpression() { * if the markings cannot be combined */ @SuppressWarnings({"unchecked", "rawtypes"}) - public static Markings combine(MarkingFunctions markingFunctions, Collection> markings) throws MarkingFunctions.Exception { - return ((MarkingFunctions) markingFunctions).combine(markings); + public static AccessExpressionMarkings combine(MarkingFunctions markingFunctions, Collection> markings) + throws MarkingFunctions.Exception { + return ((MarkingFunctions.Default) markingFunctions).combine((Collection) markings); } /** @@ -169,20 +156,14 @@ public static Markings combine(MarkingFunctions markingFunctions, Collecti * different markings * @return the merged markings */ - public static Markings mergeMarkings(MarkingFunctions markingFunctions, Markings markings1, Markings markings2) { + public static AccessExpressionMarkings mergeMarkings(MarkingFunctions.Default markingFunctions, AccessExpressionMarkings markings1, + AccessExpressionMarkings markings2) { if (markings2 != null) { if (markings1 == null) { markings1 = markings2; } else { try { - if (markings1 instanceof AccessExpressionMarkings && markings2 instanceof AccessExpressionMarkings) { - AccessExpressionMarkings aem1 = (AccessExpressionMarkings) markings1; - AccessExpressionMarkings aem2 = (AccessExpressionMarkings) markings2; - return combine(markingFunctions, List.of(aem1, aem2)); - } else { - throw new RuntimeException( - String.format("Unknown markings class %s or %s", markings1.getClass().getName(), markings1.getClass().getName())); - } + return combine(markingFunctions, List.of(markings1, markings2)); } catch (MarkingFunctions.Exception e) { throw new RuntimeException("Unable to combine markings.", e); } @@ -191,6 +172,33 @@ public static Markings mergeMarkings(MarkingFunctions markingFunctions, Ma return markings1; } + /** + * A helper routine to merge markings when merging fields of a NormalizedContentInterface + * + * @param markings1 + * markings + * @param markings2 + * different markings + * @return the merged markings + */ + public static AccessExpressionMarkings mergeMarkings(MarkingFunctions markingFunctions, Markings markings1, Markings markings2) { + if (markings2 != null) { + if (markings1 == null) { + markings1 = markings2; + } else { + if (markings1 instanceof AccessExpressionMarkings && markings2 instanceof AccessExpressionMarkings) { + AccessExpressionMarkings aem1 = (AccessExpressionMarkings) markings1; + AccessExpressionMarkings aem2 = (AccessExpressionMarkings) markings2; + return mergeMarkings((MarkingFunctions.Default) markingFunctions, aem1, aem2); + } else { + throw new RuntimeException( + String.format("Unknown markings class %s or %s", markings1.getClass().getName(), markings1.getClass().getName())); + } + } + } + return (AccessExpressionMarkings) markings1; + } + /** * As part of normalizing access expression this class is used to sort and dedupe sub-expressions in a tree set. */ diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java index 3659db33d49..267c3048c44 100644 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java @@ -104,7 +104,7 @@ public AccessExpressionMarkings combine(Collection mar @Override public ColumnVisibility translateToColumnVisibility(Markings markings) { - AccessExpression accessExpression = AccessExpressionUtil.toAccessExpression(markings); + AccessExpression accessExpression = markings.toAccessExpression(); return AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.normalize(accessExpression)); } diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/Markings.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/Markings.java index 3101a2e8354..2d8a8e7d2cd 100644 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/Markings.java +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/Markings.java @@ -1,5 +1,8 @@ package datawave.marking; +import org.apache.accumulo.access.AccessExpression; +import org.apache.accumulo.core.security.ColumnVisibility; + import com.fasterxml.jackson.annotation.JsonTypeInfo; @JsonTypeInfo(use = JsonTypeInfo.Id.CLASS) @@ -7,6 +10,10 @@ public interface Markings { T getMarkings(); + AccessExpression toAccessExpression(); + + ColumnVisibility toColumnVisibility(); + default boolean isEmpty() { return getMarkings() == null; } diff --git a/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java b/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java index ea5004f2c14..b039ec0d913 100644 --- a/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java +++ b/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java @@ -586,9 +586,10 @@ private Map buildCacheEntries() { private ColumnVisibility combineAllVisibilities() { Set visibilities = visibilityToDateFrequencies.keySet(); try { - return AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(markingFunctions, + return AccessExpressionUtil.combine(markingFunctions, visibilities.stream().map(cv -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(cv))) - .collect(Collectors.toSet()))); + .collect(Collectors.toSet())) + .toColumnVisibility(); } catch (MarkingFunctions.Exception e) { log.error("Failed to combine visibilities " + visibilities); throw new IllegalArgumentException("Failed to combine visibilities " + visibilities, e); diff --git a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java index f221923ecef..1293e12d354 100644 --- a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java +++ b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java @@ -124,7 +124,7 @@ public void setDescriptions(MetadataEntry entry, Set bw = accumuloClient.createBatchWriter(metadataTableName, bwConfig); Mutation m = new Mutation(entry.getFieldName()); for (DescriptionBase desc : descs) { - ColumnVisibility cv = AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.toAccessExpression(desc.getMarkings())); + ColumnVisibility cv = desc.getMarkings().toColumnVisibility(); m.put(ColumnFamilyConstants.COLF_DESC, new Text(entry.getDatatype()), cv, new Value(desc.getDescription().getBytes(UTF_8))); } bw.addMutation(m); @@ -157,7 +157,7 @@ public void removeDescription(MetadataEntry entry, DescriptionBase desc) throws BatchWriterConfig bwConfig = new BatchWriterConfig().setMaxMemory(10000L).setMaxLatency(100L, TimeUnit.MILLISECONDS).setMaxWriteThreads(1); bw = accumuloClient.createBatchWriter(metadataTableName, bwConfig); Mutation m = new Mutation(entry.getFieldName()); - ColumnVisibility cv = AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.toAccessExpression(desc.getMarkings())); + ColumnVisibility cv = desc.getMarkings().toColumnVisibility(); m.putDelete(ColumnFamilyConstants.COLF_DESC, new Text(entry.getDatatype()), cv); bw.addMutation(m); } finally { diff --git a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java index 02de938d7d6..15529a03013 100644 --- a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java +++ b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java @@ -53,7 +53,6 @@ import com.hazelcast.map.IMap; import com.hazelcast.spring.cache.HazelcastCacheManager; -import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.microservice.authorization.user.DatawaveUserDetails; import datawave.microservice.querymetric.config.QueryMetricTransportType; @@ -479,7 +478,7 @@ private boolean isMetricsAdministrator(DatawaveUserDetails currentUser) { private boolean canViewMetric(DatawaveUserDetails currentUser, BaseQueryMetric metric) throws Exception { boolean userCanViewMetric = true; - ColumnVisibility columnVisibility = AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.toAccessExpression(metric.getMarkings())); + ColumnVisibility columnVisibility = metric.getMarkings().toColumnVisibility(); for (DatawaveUser user : currentUser.getProxiedUsers()) { Authorizations authorizations = new Authorizations(user.getAuths().toArray(new String[0])); VisibilityEvaluator visibilityEvaluator = new VisibilityEvaluator(authorizations); diff --git a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java index 0452d37ffe1..36c195d232c 100644 --- a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java +++ b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java @@ -369,7 +369,7 @@ protected Multimap getEntries(ContentIndexingColumnBasedHan // get markings from metric, otherwise use the default markings Markings markings = updatedQueryMetric.getMarkings(); if (markings != null && !markings.isEmpty()) { - event.setVisibility(AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.toAccessExpression(markings))); + event.setVisibility(AccessExpressionUtil.toColumnVisibility(markings.toAccessExpression())); } else { event.setVisibility(this.queryMetricHandlerProperties.getDefaultMetricVisibility()); } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CompositeIngest.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CompositeIngest.java index 689c987dc16..e978ab24ae2 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CompositeIngest.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CompositeIngest.java @@ -1,7 +1,5 @@ package datawave.ingest.data.config.ingest; -import static datawave.marking.AccessExpressionUtil.mergeMarkings; - import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -31,6 +29,7 @@ import datawave.ingest.data.config.GroupedNormalizedContentInterface; import datawave.ingest.data.config.NormalizedContentInterface; import datawave.ingest.data.config.NormalizedFieldAndValue; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; @@ -386,7 +385,8 @@ public void addCompositeFields(List compositeFields, if (pos + 1 < fields.length) { addCompositeFields(compositeFields, eventFields, compositeFieldName.replace("*", replacement), replacement, (grouping == null ? newGrouping : grouping), groupingPolicy, allowMissing, fields, pos + 1, originalValue, - normalizedValue, mergeMarkings(markingFunctions, markings, value.getMarkings()), isOverloadedField); + normalizedValue, AccessExpressionUtil.mergeMarkings(markingFunctions, markings, value.getMarkings()), + isOverloadedField); } originalValue.setLength(oLen); normalizedValue.setLength(nLen); @@ -418,7 +418,7 @@ public void addCompositeFields(List compositeFields, } addCompositeFields(compositeFields, eventFields, compositeFieldName, replacement, (grouping == null ? newGrouping : grouping), groupingPolicy, allowMissing, fields, pos + 1, originalValue, normalizedValue, - mergeMarkings(markingFunctions, markings, value.getMarkings()), isOverloadedField); + AccessExpressionUtil.mergeMarkings(markingFunctions, markings, value.getMarkings()), isOverloadedField); originalValue.setLength(oLen); normalizedValue.setLength(nLen); } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomDataTypeHandler.java index 0f2f3ae4f8b..a91e435ab25 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomDataTypeHandler.java @@ -261,7 +261,7 @@ public Key createKey(String fieldName, String fieldValue, String columnQualifier protected String getColumnQualifier(RawRecordContainer event, NormalizedContentInterface value) { Markings markings = value.getMarkings(); if (markings != null && !markings.isEmpty()) { - return AccessExpressionUtil.toAccessExpression(markings).getExpression(); + return markings.toAccessExpression().getExpression(); } return flatten(event.getVisibility()); } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomErrorDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomErrorDataTypeHandler.java index 24612ed07f5..0b490dcdd22 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomErrorDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomErrorDataTypeHandler.java @@ -29,7 +29,6 @@ import datawave.ingest.mapreduce.handler.error.ErrorShardedDataTypeHandler; import datawave.ingest.mapreduce.job.BulkIngestKey; import datawave.ingest.mapreduce.job.writer.ContextWriter; -import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; @@ -78,7 +77,7 @@ public long process(KEYIN key, RawRecordContainer record, Multimap markings = value.getMarkings(); if (markings != null && !markings.isEmpty()) { - return AccessExpressionUtil.toAccessExpression(markings).getExpression(); + return markings.toAccessExpression().getExpression(); } ColumnVisibility vis = event.getVisibility(); if (vis != null) { diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDataTypeHandler.java index 1b9b6459f7d..8f3f24bb86c 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDataTypeHandler.java @@ -1337,7 +1337,7 @@ protected Text getVisibility(Markings markings, RawRecordContainer event) { if (null == markings || markings.isEmpty()) { return new Text(flatten(event.getVisibility())); } else { - AccessExpression ae = AccessExpressionUtil.normalize(AccessExpressionUtil.toAccessExpression(markings)); + AccessExpression ae = AccessExpressionUtil.normalize(markings.toAccessExpression()); return new Text(ae.getExpression().getBytes(UTF_8)); } } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorDataTypeHandler.java index a5553e4549d..f791cab06b1 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorDataTypeHandler.java @@ -135,7 +135,7 @@ public void setup(TaskAttemptContext context) { this.conf = context.getConfiguration(); - defaultVisibility = AccessExpressionUtil.toAccessExpression(markingsHelper.getDefaultMarkings()).getExpression().getBytes(UTF_8); + defaultVisibility = markingsHelper.getDefaultMarkings().toAccessExpression().getExpression().getBytes(UTF_8); // Initialize a UID builder based on the configuration uidBuilder = UID.builder(conf); @@ -318,7 +318,7 @@ private Key createKey(String row, Text colf, Text colq, byte[] vis, long ts) { protected byte[] getVisibility(RawRecordContainer event, NormalizedContentInterface value) { byte[] visibility; if (value != null && value.getMarkings() != null && !value.getMarkings().isEmpty()) { - visibility = AccessExpressionUtil.toAccessExpression(value.getMarkings()).getExpression().getBytes(UTF_8); + visibility = value.getMarkings().toAccessExpression().getExpression().getBytes(UTF_8); } else if (event.getVisibility() != null) { visibility = flatten(event.getVisibility()); } else { diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java index 5c406337210..aa22efdb471 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java @@ -43,7 +43,6 @@ import datawave.ingest.mapreduce.handler.shard.ShardedDataTypeHandler; import datawave.ingest.mapreduce.job.BulkIngestKey; import datawave.ingest.mapreduce.job.writer.ContextWriter; -import datawave.marking.AccessExpressionUtil; /** * Handler that take events with processing errors or fatal errors and dumps them into a processing error table. This table will be used for subsequent @@ -158,7 +157,7 @@ public void setup(TaskAttemptContext context) { setShardDictionaryIndexTableName(tableName == null ? null : new Text(tableName)); try { - defaultVisibility = AccessExpressionUtil.toAccessExpression(markingsHelper.getDefaultMarkings()).getExpression().getBytes(UTF_8); + defaultVisibility = markingsHelper.getDefaultMarkings().toAccessExpression().getExpression().getBytes(UTF_8); } catch (Exception e) { throw new IllegalArgumentException("Failed to parse security marking configuration", e); } @@ -349,7 +348,7 @@ public static void getStackTrace(DataOutputBuffer buffer, Throwable e) { public byte[] getVisibility(RawRecordContainer event, NormalizedContentInterface value) { byte[] visibility; if (value != null && value.getMarkings() != null && !value.getMarkings().isEmpty()) { - visibility = AccessExpressionUtil.toAccessExpression(value.getMarkings()).getExpression().getBytes(UTF_8); + visibility = value.getMarkings().toAccessExpression().getExpression().getBytes(UTF_8); } else if (event.getVisibility() != null) { visibility = flatten(event.getVisibility()); } else { diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java index ff27490482f..c770302d20b 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java @@ -1135,7 +1135,7 @@ public int getOffsetForYearIndex(String shard) { public byte[] getVisibility(RawRecordContainer event, NormalizedContentInterface value) { Markings markings = value.getMarkings(); if (markings != null && !markings.isEmpty()) { - return AccessExpressionUtil.toAccessExpression(markings).getExpression().getBytes(UTF_8); + return markings.toAccessExpression().getExpression().getBytes(UTF_8); } return flatten(event.getVisibility()); } diff --git a/warehouse/query-core/src/main/java/datawave/core/iterators/FieldIndexCountingIteratorPerVisibility.java b/warehouse/query-core/src/main/java/datawave/core/iterators/FieldIndexCountingIteratorPerVisibility.java index b6bcfda1961..d0af39e1f5c 100644 --- a/warehouse/query-core/src/main/java/datawave/core/iterators/FieldIndexCountingIteratorPerVisibility.java +++ b/warehouse/query-core/src/main/java/datawave/core/iterators/FieldIndexCountingIteratorPerVisibility.java @@ -567,10 +567,11 @@ private Map buildReturnKeys() { ColumnVisibility cv = null; try { // Calculate the columnVisibility for this key from the combiner. - cv = AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(MarkingFunctions.Factory.createMarkingFunctions(), + cv = AccessExpressionUtil.combine(MarkingFunctions.Factory.createMarkingFunctions(), columnVisibilities.stream() .map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))) - .collect(Collectors.toSet()))); + .collect(Collectors.toSet())) + .toColumnVisibility(); } catch (Exception e) { log.error("Could not create combined columnVisibility for the count", e); return null; diff --git a/warehouse/query-core/src/main/java/datawave/core/iterators/GlobalIndexDateSummaryIterator.java b/warehouse/query-core/src/main/java/datawave/core/iterators/GlobalIndexDateSummaryIterator.java index 2cda3b801e8..328a13c261f 100644 --- a/warehouse/query-core/src/main/java/datawave/core/iterators/GlobalIndexDateSummaryIterator.java +++ b/warehouse/query-core/src/main/java/datawave/core/iterators/GlobalIndexDateSummaryIterator.java @@ -258,11 +258,11 @@ public Map getKeyValues() throws IOException { Set columnVisibilities = this.columnVisibilitiesMap.get(datatype); // Note that the access controls found in the combined ColumnVisibility will be pulled out appropriately here - ColumnVisibility cv = AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine( - MarkingFunctions.Factory.createMarkingFunctions(), + ColumnVisibility cv = AccessExpressionUtil.combine(MarkingFunctions.Factory.createMarkingFunctions(), columnVisibilities.stream() .map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))) - .collect(Collectors.toSet()))); + .collect(Collectors.toSet())) + .toColumnVisibility(); // Create a new Key compatible with the shardIndex key format Key k = new Key(this.fieldValue, this.fieldName, this.date + '\0' + datatype, new String(cv.getExpression())); diff --git a/warehouse/query-core/src/main/java/datawave/core/iterators/ResultCountingIterator.java b/warehouse/query-core/src/main/java/datawave/core/iterators/ResultCountingIterator.java index ffa894aff88..77901e4499d 100644 --- a/warehouse/query-core/src/main/java/datawave/core/iterators/ResultCountingIterator.java +++ b/warehouse/query-core/src/main/java/datawave/core/iterators/ResultCountingIterator.java @@ -211,9 +211,9 @@ public Value getTopValue() { ColumnVisibility cv = null; try { - cv = AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(MarkingFunctions.Factory.createMarkingFunctions(), - columnVisibilities.stream().map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))) - .collect(Collectors.toSet()))); + cv = AccessExpressionUtil.combine(MarkingFunctions.Factory.createMarkingFunctions(), columnVisibilities.stream() + .map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))).collect(Collectors.toSet())) + .toColumnVisibility(); } catch (MarkingFunctions.Exception e) { log.error("Could not create combined columnVisibility for the count", e); return null; diff --git a/warehouse/query-core/src/main/java/datawave/query/attributes/AttributeBag.java b/warehouse/query-core/src/main/java/datawave/query/attributes/AttributeBag.java index 9d69e02cd79..597b99b18c5 100644 --- a/warehouse/query-core/src/main/java/datawave/query/attributes/AttributeBag.java +++ b/warehouse/query-core/src/main/java/datawave/query/attributes/AttributeBag.java @@ -81,8 +81,10 @@ private void updateMetadata() { protected ColumnVisibility combineAndSetColumnVisibilities(Collection>> attributes) throws Exception { Collection accessExpressions = attributes.stream().map(attr -> AccessExpressionUtil.toAccessExpression(attr.getColumnVisibility())) .collect(Collectors.toSet()); - return AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(MarkingFunctions.Factory.createMarkingFunctions(), - accessExpressions.stream().map(AccessExpressionUtil::toAccessExpressionMarkings).collect(Collectors.toList()))); + return AccessExpressionUtil + .combine(MarkingFunctions.Factory.createMarkingFunctions(), + accessExpressions.stream().map(AccessExpressionUtil::toAccessExpressionMarkings).collect(Collectors.toList())) + .toColumnVisibility(); } private long updateTimestamps() { diff --git a/warehouse/query-core/src/main/java/datawave/query/common/grouping/GroupingUtils.java b/warehouse/query-core/src/main/java/datawave/query/common/grouping/GroupingUtils.java index f63cd8dcbd6..42fea0584b3 100644 --- a/warehouse/query-core/src/main/java/datawave/query/common/grouping/GroupingUtils.java +++ b/warehouse/query-core/src/main/java/datawave/query/common/grouping/GroupingUtils.java @@ -47,8 +47,9 @@ public enum AverageAggregatorWriteFormat { */ public static ColumnVisibility combineVisibilities(Collection expressions, MarkingFunctions markingFunctions, boolean failOnError) { try { - return AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(markingFunctions, - expressions.stream().map(AccessExpressionUtil::toAccessExpressionMarkings).collect(Collectors.toList()))); + return AccessExpressionUtil + .combine(markingFunctions, expressions.stream().map(AccessExpressionUtil::toAccessExpressionMarkings).collect(Collectors.toList())) + .toColumnVisibility(); } catch (MarkingFunctions.Exception e) { if (failOnError) { throw new IllegalArgumentException("Unable to combine access expressions: " + expressions, e); diff --git a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java index dd5f002eb33..fb95db77f9e 100644 --- a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java +++ b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java @@ -163,10 +163,11 @@ private DiscoveredThing aggregate(Collection termEntries) { } else { // Otherwise, combine the visibilities, and return the aggregated result. try { - ColumnVisibility visibility = AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(markingFunctions, + ColumnVisibility visibility = AccessExpressionUtil.combine(markingFunctions, visibilities.stream() .map(cv -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(cv))) - .collect(Collectors.toSet()))); + .collect(Collectors.toSet())) + .toColumnVisibility(); MapWritable countsByVis = new MapWritable(); visibilityToCounts.forEach((key, value) -> countsByVis.put(new Text(key), new LongWritable(value))); String normalizedVis = AccessExpressionUtil.normalize(AccessExpressionUtil.toAccessExpression(visibility)).getExpression(); diff --git a/warehouse/query-core/src/main/java/datawave/query/discovery/TermInfoAggregation.java b/warehouse/query-core/src/main/java/datawave/query/discovery/TermInfoAggregation.java index 6a5323103aa..d9138e3cdfe 100644 --- a/warehouse/query-core/src/main/java/datawave/query/discovery/TermInfoAggregation.java +++ b/warehouse/query-core/src/main/java/datawave/query/discovery/TermInfoAggregation.java @@ -110,10 +110,11 @@ public DiscoveredThing apply(Collection from) { ColumnVisibility columnVisibility = null; try { - columnVisibility = AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(MarkingFunctions.Factory.createMarkingFunctions(), + columnVisibility = AccessExpressionUtil.combine(MarkingFunctions.Factory.createMarkingFunctions(), columnVisibilities.stream() .map(cv -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(cv))) - .collect(Collectors.toSet()))); + .collect(Collectors.toSet())) + .toColumnVisibility(); } catch (Exception e) { log.warn("Invalid columnvisibility after combining!", e); diff --git a/warehouse/query-core/src/main/java/datawave/query/iterators/FieldIndexCountingIterator.java b/warehouse/query-core/src/main/java/datawave/query/iterators/FieldIndexCountingIterator.java index bf22f948828..83f4ac5a741 100644 --- a/warehouse/query-core/src/main/java/datawave/query/iterators/FieldIndexCountingIterator.java +++ b/warehouse/query-core/src/main/java/datawave/query/iterators/FieldIndexCountingIterator.java @@ -589,9 +589,9 @@ private Key buildReturnKey() { } ColumnVisibility cv; try { - cv = AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(MarkingFunctions.Factory.createMarkingFunctions(), - columnVisibilities.stream().map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))) - .collect(Collectors.toSet()))); + cv = AccessExpressionUtil.combine(MarkingFunctions.Factory.createMarkingFunctions(), columnVisibilities.stream() + .map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))).collect(Collectors.toSet())) + .toColumnVisibility(); } catch (MarkingFunctions.Exception e) { log.error("Could not combine visibilities: " + visibilitySet + " " + e); return null; diff --git a/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java b/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java index 7f32cd69d33..810ebc8a5c5 100644 --- a/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java +++ b/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java @@ -471,9 +471,9 @@ private void addHitsForFunction(String field, ASTFunctionNode node, HitListArith } try { - cv = AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(MarkingFunctionsFactory.createMarkingFunctions(), - cvs.stream().map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))) - .collect(Collectors.toSet()))); + cv = AccessExpressionUtil.combine(MarkingFunctionsFactory.createMarkingFunctions(), cvs.stream() + .map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))).collect(Collectors.toSet())) + .toColumnVisibility(); } catch (MarkingFunctions.Exception e) { log.error("Failed to combine column visibilities while generating HIT_TERM for phrase function for field [" + field + "]"); log.error("msg: ", e); diff --git a/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java b/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java index 4cf3344e7ae..502f3e68ee3 100644 --- a/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java +++ b/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java @@ -293,8 +293,10 @@ public Attribute joinAttributes(String compositeName, Collection if (log.isDebugEnabled()) { log.debug("dataList is " + dataList); } - ColumnVisibility combinedColumnVisibility = AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(this.markingFunctions, - accessExpressions.stream().map(AccessExpressionUtil::toAccessExpressionMarkings).collect(Collectors.toList()))); + ColumnVisibility combinedColumnVisibility = AccessExpressionUtil + .combine(this.markingFunctions, + accessExpressions.stream().map(AccessExpressionUtil::toAccessExpressionMarkings).collect(Collectors.toList())) + .toColumnVisibility(); metadata = new Key(metadata.getRow(), metadata.getColumnFamily(), new Text(), combinedColumnVisibility, timestamp); if (dataList.size() == 1) { return this.attrFactory.create(compositeName, dataList.get(0), metadata, toKeep, true); diff --git a/warehouse/query-core/src/main/java/datawave/query/table/parser/EventKeyValueFactory.java b/warehouse/query-core/src/main/java/datawave/query/table/parser/EventKeyValueFactory.java index 88b012323bb..0ada017121a 100644 --- a/warehouse/query-core/src/main/java/datawave/query/table/parser/EventKeyValueFactory.java +++ b/warehouse/query-core/src/main/java/datawave/query/table/parser/EventKeyValueFactory.java @@ -1,8 +1,6 @@ package datawave.query.table.parser; import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.ColumnVisibility; import datawave.marking.AccessExpressionUtil; @@ -12,7 +10,7 @@ public class EventKeyValueFactory { - public static EventKeyValue parse(Key key, Value value, Authorizations auths, MarkingFunctions markingFunctions) throws MarkingFunctions.Exception { + public static EventKeyValue parse(Key key) throws MarkingFunctions.Exception { if (null == key) throw new IllegalArgumentException("Cannot pass null key to EventKeyValueFactory"); diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountAggregatingIterator.java b/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountAggregatingIterator.java index a20b7d62baf..afc3facd7d5 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountAggregatingIterator.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountAggregatingIterator.java @@ -154,10 +154,11 @@ public Object getAggregatedEvent() { private ColumnVisibility getCombinedColumnVisibility() { try { - return AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(markingFunctions, + return AccessExpressionUtil.combine(markingFunctions, columnVisibilities.stream() .map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))) - .collect(Collectors.toSet()))); + .collect(Collectors.toSet())) + .toColumnVisibility(); } catch (Exception e) { log.error("Could not combine columnVisibilities for the count", e); return null; diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountResultPostprocessor.java b/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountResultPostprocessor.java index 683cad0604b..6dc38992c2f 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountResultPostprocessor.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountResultPostprocessor.java @@ -72,10 +72,11 @@ public void apply(List results) { if (success) { ColumnVisibility columnVisibility = null; try { - columnVisibility = AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(markingFunctions, + columnVisibility = AccessExpressionUtil.combine(markingFunctions, columnVisibilities.stream() .map(cv -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(cv))) - .collect(Collectors.toSet()))); + .collect(Collectors.toSet())) + .toColumnVisibility(); } catch (Exception e) { log.error("Could not create combined columnVisibilities for the count", e); } diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java index 1464c3f3d8a..56d2ea70bfa 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java @@ -509,10 +509,11 @@ public ColumnVisibility getColumnVisibility() { for (Text t : this.uniqueVisibilities) { columnVisibilities.add(new ColumnVisibility(t)); } - return AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.combine(tupleMarkingFunctions, + return AccessExpressionUtil.combine(tupleMarkingFunctions, columnVisibilities.stream() .map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))) - .collect(Collectors.toList()))); + .collect(Collectors.toList())) + .toColumnVisibility(); } catch (MarkingFunctions.Exception e) { logger.error("Could not create combined column visibility for the count", e); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java index 93f74e416d3..18be701e996 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java @@ -102,8 +102,7 @@ public EventBase transform(Entry entry) { if (getContentFieldNames().contains(fieldName)) { value = baseUid; } - values.add(this.makeField(fieldName, fieldMarkings, AccessExpressionUtil.toAccessExpression(fieldMarkings).getExpression(), - entry.getKey().getTimestamp(), value)); + values.add(this.makeField(fieldName, fieldMarkings, fieldMarkings.toAccessExpression().getExpression(), entry.getKey().getTimestamp(), value)); } } diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java index 0ac3828c3e5..2505a335f66 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java @@ -129,7 +129,7 @@ protected Collection buildFacets(Key documentKey, String f FieldCardinalityBase fc = this.responseObjectFactory.getFieldCardinality(); fc.setField(v.getFieldName()); fc.setMarkings(AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(attr.getColumnVisibility()))); - fc.setColumnVisibility(AccessExpressionUtil.normalize(AccessExpressionUtil.toAccessExpression(fc.getMarkings())).getExpression()); + fc.setColumnVisibility(AccessExpressionUtil.normalize(fc.getMarkings().toAccessExpression()).getExpression()); fc.setLower(v.getFloorValue()); fc.setUpper(v.getCeilingValue()); fc.setCardinality(v.getEstimate().cardinality()); diff --git a/web-services/client/src/main/java/datawave/webservice/modification/DefaultModificationRequest.java b/web-services/client/src/main/java/datawave/webservice/modification/DefaultModificationRequest.java index d1933578cbe..542c764e325 100644 --- a/web-services/client/src/main/java/datawave/webservice/modification/DefaultModificationRequest.java +++ b/web-services/client/src/main/java/datawave/webservice/modification/DefaultModificationRequest.java @@ -84,7 +84,7 @@ public Map getFieldMarkings() { } public void setFieldMarkings(Map fieldMarkings) { - this.fieldMarkings = (fieldMarkings == null ? new HashMap() : new HashMap(fieldMarkings)); + this.fieldMarkings = (fieldMarkings == null ? new HashMap<>() : new HashMap<>(fieldMarkings)); } public Map getOldFieldMarkings() { @@ -94,7 +94,7 @@ public Map getOldFieldMarkings() { } public void setOldFieldMarkings(Map oldFieldMarkings) { - this.oldFieldMarkings = (oldFieldMarkings == null ? new HashMap() : new HashMap(oldFieldMarkings)); + this.oldFieldMarkings = (oldFieldMarkings == null ? new HashMap<>() : new HashMap<>(oldFieldMarkings)); } public String getOldFieldValue() { diff --git a/web-services/metrics/src/main/java/datawave/query/metrics/ShardTableQueryMetricHandler.java b/web-services/metrics/src/main/java/datawave/query/metrics/ShardTableQueryMetricHandler.java index 0fea43042fd..f19e4f91456 100644 --- a/web-services/metrics/src/main/java/datawave/query/metrics/ShardTableQueryMetricHandler.java +++ b/web-services/metrics/src/main/java/datawave/query/metrics/ShardTableQueryMetricHandler.java @@ -75,7 +75,6 @@ import datawave.ingest.mapreduce.job.BulkIngestKey; import datawave.ingest.mapreduce.job.writer.LiveContextWriter; import datawave.ingest.table.config.TableConfigHelper; -import datawave.marking.AccessExpressionUtil; import datawave.marking.Markings; import datawave.microservice.query.Query; import datawave.microservice.query.QueryImpl; @@ -284,7 +283,7 @@ private Multimap getEntries(AbstractColumnBasedHandler event.setVisibility(new ColumnVisibility(DEFAULT_SECURITY_MARKING)); } else { try { - event.setVisibility(AccessExpressionUtil.toColumnVisibility(markings)); + event.setVisibility(markings.toColumnVisibility()); } catch (Exception e) { log.error(e.getMessage(), e); event.setVisibility(new ColumnVisibility(DEFAULT_SECURITY_MARKING)); From 480c1ec662a813d73d41b8ab5127728192ef30f1 Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Tue, 21 Apr 2026 12:05:53 -0400 Subject: [PATCH 23/63] marking class updates --- .../cachedresults/CacheableQueryRowImpl.java | 5 +- .../marking/AccessExpressionUtil.java | 109 ++++-------------- .../marking/FlattenedVisibilityCache.java | 2 +- .../datawave/marking/MarkingFunctions.java | 37 +++++- .../FrequencyMetadataAggregator.java | 8 +- .../data/config/ingest/CompositeIngest.java | 25 ++-- .../data/config/ingest/VirtualIngest.java | 24 ++-- .../handler/edge/define/EdgeDataBundle.java | 8 +- .../util/keyword/DefaultTagCloudUtils.java | 8 +- ...eldIndexCountingIteratorPerVisibility.java | 8 +- .../GlobalIndexDateSummaryIterator.java | 8 +- .../iterators/ResultCountingIterator.java | 11 +- .../query/attributes/AttributeBag.java | 15 ++- .../query/common/grouping/GroupingUtils.java | 3 +- .../query/discovery/DiscoveryIterator.java | 11 +- .../query/discovery/TermInfoAggregation.java | 8 +- .../iterators/FieldIndexCountingIterator.java | 11 +- .../query/jexl/DatawaveInterpreter.java | 13 +-- .../query/predicate/ValueToAttributes.java | 7 +- .../shard/CountAggregatingIterator.java | 8 +- .../shard/CountResultPostprocessor.java | 8 +- .../shard/FieldIndexCountQueryLogic.java | 25 ++-- .../EdgeQueryTransformerSupport.java | 3 +- .../transformer/EventQueryTransformer.java | 13 +-- .../query/transformer/FacetedTransformer.java | 9 +- .../FieldIndexCountQueryTransformer.java | 2 +- .../predicate/ValueToAttributesTest.java | 3 +- .../tables/ssdeep/SSDeepIngestQueryTest.java | 3 +- .../AnnotationHitsTransformerTest.java | 3 +- 29 files changed, 162 insertions(+), 236 deletions(-) diff --git a/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java b/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java index 43c2e08fa68..aeedbe831b5 100644 --- a/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java +++ b/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java @@ -19,7 +19,6 @@ import datawave.data.type.Type; import datawave.marking.AccessExpressionMarkings; -import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.webservice.query.cachedresults.CacheableQueryRow; @@ -69,7 +68,7 @@ public void addColumn(String columnName, TypedValue columnTypedValue, Markings datawaveType, String columnName, TypedVa if (!currMarkings.equals(markings)) { try { - Markings combinedMarkings = AccessExpressionUtil.combine(markingFunctions, List.of(currMarkings, markings)); + Markings combinedMarkings = markingFunctions.combine(currMarkings, markings); // use combined marking as new markings columnMarkingsMap.put(columnName, combinedMarkings); diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionUtil.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionUtil.java index 7e2bd60872e..55464281d8e 100644 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionUtil.java +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionUtil.java @@ -6,8 +6,6 @@ import static org.apache.accumulo.access.ParsedAccessExpression.ExpressionType.AND; import static org.apache.accumulo.access.ParsedAccessExpression.ExpressionType.AUTHORIZATION; -import java.util.Collection; -import java.util.List; import java.util.SortedSet; import java.util.TreeSet; @@ -129,76 +127,6 @@ public static AccessExpression emptyExpression() { return EMPTY_EXPRESSION; } - /** - * Helper to call {@link MarkingFunctions#combine(Collection)} with a wildcard-typed {@link MarkingFunctions}. This avoids the wildcard-capture issue that - * arises when callers hold a {@code MarkingFunctions} reference. - * - * @param markingFunctions - * the marking functions instance - * @param markings - * the markings to combine - * @return the combined markings - * @throws MarkingFunctions.Exception - * if the markings cannot be combined - */ - @SuppressWarnings({"unchecked", "rawtypes"}) - public static AccessExpressionMarkings combine(MarkingFunctions markingFunctions, Collection> markings) - throws MarkingFunctions.Exception { - return ((MarkingFunctions.Default) markingFunctions).combine((Collection) markings); - } - - /** - * A helper routine to merge markings when merging fields of a NormalizedContentInterface - * - * @param markings1 - * markings - * @param markings2 - * different markings - * @return the merged markings - */ - public static AccessExpressionMarkings mergeMarkings(MarkingFunctions.Default markingFunctions, AccessExpressionMarkings markings1, - AccessExpressionMarkings markings2) { - if (markings2 != null) { - if (markings1 == null) { - markings1 = markings2; - } else { - try { - return combine(markingFunctions, List.of(markings1, markings2)); - } catch (MarkingFunctions.Exception e) { - throw new RuntimeException("Unable to combine markings.", e); - } - } - } - return markings1; - } - - /** - * A helper routine to merge markings when merging fields of a NormalizedContentInterface - * - * @param markings1 - * markings - * @param markings2 - * different markings - * @return the merged markings - */ - public static AccessExpressionMarkings mergeMarkings(MarkingFunctions markingFunctions, Markings markings1, Markings markings2) { - if (markings2 != null) { - if (markings1 == null) { - markings1 = markings2; - } else { - if (markings1 instanceof AccessExpressionMarkings && markings2 instanceof AccessExpressionMarkings) { - AccessExpressionMarkings aem1 = (AccessExpressionMarkings) markings1; - AccessExpressionMarkings aem2 = (AccessExpressionMarkings) markings2; - return mergeMarkings((MarkingFunctions.Default) markingFunctions, aem1, aem2); - } else { - throw new RuntimeException( - String.format("Unknown markings class %s or %s", markings1.getClass().getName(), markings1.getClass().getName())); - } - } - } - return (AccessExpressionMarkings) markings1; - } - /** * As part of normalizing access expression this class is used to sort and dedupe sub-expressions in a tree set. */ @@ -257,7 +185,7 @@ public int hashCode() { * This method helps with the flattening aspect of normalization by recursing down as far as possible the parse tree in the case when the expression type is * the same. As long as the type is the same in the sub expression, keep using the same set. */ - public static void flatten(ParsedAccessExpression.ExpressionType parentType, ParsedAccessExpression parsed, + private static void flatten(ParsedAccessExpression.ExpressionType parentType, ParsedAccessExpression parsed, SortedSet normalizedExpressions) { if (parsed.getType() == parentType) { for (var child : parsed.getChildren()) { @@ -304,25 +232,30 @@ public static NormalizedExpression normalize(ParsedAccessExpression parsed) { if (normalizedChildren.size() == 1) { return normalizedChildren.first(); } else { - String operator = parsed.getType() == AND ? "&" : "|"; - String sep = ""; + StringBuilder builder = getStringBuilder(parsed, normalizedChildren); - StringBuilder builder = new StringBuilder(); + return new NormalizedExpression(builder.toString(), parsed.getType()); + } + } + } - for (var child : normalizedChildren) { - builder.append(sep); - if (child.type == AUTHORIZATION) { - builder.append(child.expression); - } else { - builder.append("("); - builder.append(child.expression); - builder.append(")"); - } - sep = operator; - } + private static StringBuilder getStringBuilder(ParsedAccessExpression parsed, TreeSet normalizedChildren) { + String operator = parsed.getType() == AND ? "&" : "|"; + String sep = ""; - return new NormalizedExpression(builder.toString(), parsed.getType()); + StringBuilder builder = new StringBuilder(); + + for (var child : normalizedChildren) { + builder.append(sep); + if (child.type == AUTHORIZATION) { + builder.append(child.expression); + } else { + builder.append("("); + builder.append(child.expression); + builder.append(")"); } + sep = operator; } + return builder; } } diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/FlattenedVisibilityCache.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/FlattenedVisibilityCache.java index 85e1697cc26..2b77662c99b 100644 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/FlattenedVisibilityCache.java +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/FlattenedVisibilityCache.java @@ -13,7 +13,7 @@ * */ public class FlattenedVisibilityCache { - private static Map flattenedVisCache = Collections.synchronizedMap(new HashMap<>()); + private static final Map flattenedVisCache = Collections.synchronizedMap(new HashMap<>()); /** * Create a flattened visibility, using the cache if possible diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java index 267c3048c44..adeaed2d5c1 100644 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java @@ -5,6 +5,7 @@ import java.nio.charset.StandardCharsets; import java.util.Collection; import java.util.HashSet; +import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -27,7 +28,9 @@ public interface MarkingFunctions> { ColumnVisibility combineVisibilities(Collection visibilities) throws MarkingFunctions.Exception; - T combine(Collection markings) throws MarkingFunctions.Exception; + T combine(Collection> markings) throws MarkingFunctions.Exception; + + T combine(Markings markings1, Markings markings2) throws MarkingFunctions.Exception; ColumnVisibility translateToColumnVisibility(Markings markings) throws MarkingFunctions.Exception; @@ -73,9 +76,16 @@ public ColumnVisibility combineVisibilities(Collection visibil } @Override - public AccessExpressionMarkings combine(Collection markings) { + public AccessExpressionMarkings combine(Collection> markings) { + + Set uniqueMarkings = new HashSet<>(); - Set uniqueMarkings = new HashSet<>(markings); + for (Markings marking : markings) { + if (!(marking instanceof AccessExpressionMarkings)) { + throw new RuntimeException(String.format("Unknown markings class %s", marking.getClass().getName())); + } + uniqueMarkings.add((AccessExpressionMarkings) marking); + } if (uniqueMarkings.isEmpty()) { return EMPTY_MARKINGS; @@ -85,7 +95,7 @@ public AccessExpressionMarkings combine(Collection mar return uniqueMarkings.stream().findFirst().get(); } - Set uniqueExpressions = markings.stream().map(AccessExpressionMarkings::getMarkings).collect(Collectors.toSet()); + Set uniqueExpressions = uniqueMarkings.stream().map(AccessExpressionMarkings::getMarkings).collect(Collectors.toSet()); // filter out any empty expressions and concatenate with '&' // @formatter:off @@ -102,6 +112,25 @@ public AccessExpressionMarkings combine(Collection mar return AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression(AccessExpressionUtil.normalize(expression).expression)).build(); } + @Override + public AccessExpressionMarkings combine(Markings markings1, Markings markings2) { + if (markings2 != null) { + if (markings1 == null) { + markings1 = markings2; + } else { + if (markings1 instanceof AccessExpressionMarkings && markings2 instanceof AccessExpressionMarkings) { + AccessExpressionMarkings aem1 = (AccessExpressionMarkings) markings1; + AccessExpressionMarkings aem2 = (AccessExpressionMarkings) markings2; + return combine(List.of(aem1, aem2)); + } else { + throw new RuntimeException( + String.format("Unknown markings class %s or %s", markings1.getClass().getName(), markings2.getClass().getName())); + } + } + } + return markings1 == null ? EMPTY_MARKINGS : (AccessExpressionMarkings) markings1; + } + @Override public ColumnVisibility translateToColumnVisibility(Markings markings) { AccessExpression accessExpression = markings.toAccessExpression(); diff --git a/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java b/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java index b039ec0d913..3e07a1622db 100644 --- a/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java +++ b/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java @@ -31,6 +31,7 @@ import com.google.common.base.Splitter; import datawave.iterator.ReducingIterator; +import datawave.marking.AccessExpressionMarkings; import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.query.model.DateFrequencyMap; @@ -56,7 +57,7 @@ public class FrequencyMetadataAggregator extends WrappingIterator implements Opt private static final Logger log = Logger.getLogger(FrequencyMetadataAggregator.class); private static final String NULL_BYTE = "\0"; - private static final MarkingFunctions markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); + private static final MarkingFunctions markingFunctions = new MarkingFunctions.Default(); private boolean combineVisibilities; private String columnsOption; @@ -586,13 +587,12 @@ private Map buildCacheEntries() { private ColumnVisibility combineAllVisibilities() { Set visibilities = visibilityToDateFrequencies.keySet(); try { - return AccessExpressionUtil.combine(markingFunctions, + return markingFunctions.combine( visibilities.stream().map(cv -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(cv))) .collect(Collectors.toSet())) .toColumnVisibility(); } catch (MarkingFunctions.Exception e) { - log.error("Failed to combine visibilities " + visibilities); - throw new IllegalArgumentException("Failed to combine visibilities " + visibilities, e); + throw new RuntimeException("Could not combine markings", e); } } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CompositeIngest.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CompositeIngest.java index e978ab24ae2..6646037dc02 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CompositeIngest.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CompositeIngest.java @@ -29,7 +29,6 @@ import datawave.ingest.data.config.GroupedNormalizedContentInterface; import datawave.ingest.data.config.NormalizedContentInterface; import datawave.ingest.data.config.NormalizedFieldAndValue; -import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; @@ -383,10 +382,14 @@ public void addCompositeFields(List compositeFields, normalizedValue.append(separator); normalizedValue.append(value.getIndexedFieldValue()); if (pos + 1 < fields.length) { - addCompositeFields(compositeFields, eventFields, compositeFieldName.replace("*", replacement), replacement, - (grouping == null ? newGrouping : grouping), groupingPolicy, allowMissing, fields, pos + 1, originalValue, - normalizedValue, AccessExpressionUtil.mergeMarkings(markingFunctions, markings, value.getMarkings()), - isOverloadedField); + try { + addCompositeFields(compositeFields, eventFields, compositeFieldName.replace("*", replacement), replacement, + (grouping == null ? newGrouping : grouping), groupingPolicy, allowMissing, fields, pos + 1, + originalValue, normalizedValue, markingFunctions.combine(markings, value.getMarkings()), + isOverloadedField); + } catch (MarkingFunctions.Exception e) { + throw new RuntimeException("Could not combine markings", e); + } } originalValue.setLength(oLen); normalizedValue.setLength(nLen); @@ -406,7 +409,7 @@ public void addCompositeFields(List compositeFields, continue; } } - if (!isOverloadedField || (isOverloadedField && pos == 0)) { + if (!isOverloadedField || pos == 0) { originalValue.append(separator); originalValue.append(value.getEventFieldValue()); } @@ -416,9 +419,13 @@ public void addCompositeFields(List compositeFields, } else { normalizedValue.append(value.getIndexedFieldValue()); } - addCompositeFields(compositeFields, eventFields, compositeFieldName, replacement, (grouping == null ? newGrouping : grouping), - groupingPolicy, allowMissing, fields, pos + 1, originalValue, normalizedValue, - AccessExpressionUtil.mergeMarkings(markingFunctions, markings, value.getMarkings()), isOverloadedField); + try { + addCompositeFields(compositeFields, eventFields, compositeFieldName, replacement, (grouping == null ? newGrouping : grouping), + groupingPolicy, allowMissing, fields, pos + 1, originalValue, normalizedValue, + markingFunctions.combine(markings, value.getMarkings()), isOverloadedField); + } catch (MarkingFunctions.Exception e) { + throw new RuntimeException("Could not combine markings", e); + } originalValue.setLength(oLen); normalizedValue.setLength(nLen); } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java index 4baf799f293..e02a757db21 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java @@ -1,7 +1,5 @@ package datawave.ingest.data.config.ingest; -import static datawave.marking.AccessExpressionUtil.mergeMarkings; - import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -578,10 +576,14 @@ public void addVirtualFields(List virtualFields, Mul normalizedValue.append(endSeparator); // recurse for the next virtual field definition segment - addVirtualFields(virtualFields, eventFields, groupings, virtualFieldName.replace("*", replacement), replacement, - (grouping == null ? newGrouping : grouping), groupingPolicy, allowMissing, fields, pos + 1, - this.defaultStartSeparator, this.defaultEndSeparator, originalValue, normalizedValue, - mergeMarkings(markingFunctions, markings, value.getMarkings())); + try { + addVirtualFields(virtualFields, eventFields, groupings, virtualFieldName.replace("*", replacement), replacement, + (grouping == null ? newGrouping : grouping), groupingPolicy, allowMissing, fields, pos + 1, + this.defaultStartSeparator, this.defaultEndSeparator, originalValue, normalizedValue, + markingFunctions.combine(markings, value.getMarkings())); + } catch (MarkingFunctions.Exception e) { + throw new RuntimeException("Could not combine markings", e); + } // reset the values to the original length originalValue.setLength(oLen); @@ -617,9 +619,13 @@ public void addVirtualFields(List virtualFields, Mul normalizedValue.append(endSeparator); // recurse on the next virtual field segment - addVirtualFields(virtualFields, eventFields, groupings, virtualFieldName, replacement, (grouping == null ? newGrouping : grouping), - groupingPolicy, allowMissing, fields, pos + 1, this.defaultStartSeparator, this.defaultEndSeparator, originalValue, - normalizedValue, mergeMarkings(markingFunctions, markings, value.getMarkings())); + try { + addVirtualFields(virtualFields, eventFields, groupings, virtualFieldName, replacement, (grouping == null ? newGrouping : grouping), + groupingPolicy, allowMissing, fields, pos + 1, this.defaultStartSeparator, this.defaultEndSeparator, originalValue, + normalizedValue, markingFunctions.combine(markings, value.getMarkings())); + } catch (MarkingFunctions.Exception e) { + throw new RuntimeException("Could not combine markings", e); + } // reset the values to the original length originalValue.setLength(oLen); diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/EdgeDataBundle.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/EdgeDataBundle.java index a59959014f4..28ba422499c 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/EdgeDataBundle.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/EdgeDataBundle.java @@ -1,7 +1,5 @@ package datawave.ingest.mapreduce.handler.edge.define; -import static datawave.marking.AccessExpressionUtil.mergeMarkings; - import java.io.IOException; import java.util.Calendar; import java.util.List; @@ -96,7 +94,11 @@ public EdgeDataBundle(EdgeDefinition edgeDef, NormalizedContentInterface ifaceSo // even though event, etc. references are saved above, passing in the event // prevents future bug this.initFieldMasking(helper, event); - this.markings = mergeMarkings(markingFunctions, getSource().getMarkings(), getSink().getMarkings()); + try { + this.markings = markingFunctions.combine(getSource().getMarkings(), getSink().getMarkings()); + } catch (MarkingFunctions.Exception e) { + throw new RuntimeException("Could not combine markings", e); + } } private int getHour(long time) { diff --git a/warehouse/keyword-common/src/main/java/datawave/util/keyword/DefaultTagCloudUtils.java b/warehouse/keyword-common/src/main/java/datawave/util/keyword/DefaultTagCloudUtils.java index 901a12a4519..478dac7c2f5 100644 --- a/warehouse/keyword-common/src/main/java/datawave/util/keyword/DefaultTagCloudUtils.java +++ b/warehouse/keyword-common/src/main/java/datawave/util/keyword/DefaultTagCloudUtils.java @@ -30,13 +30,9 @@ public Markings generateCombinedVisibility(Set visibilities) { if (visibilities == null || visibilities.isEmpty()) { return null; } - Set> markings = visibilities.stream().map(v -> AccessExpressionUtil.toAccessExpressionMarkings(ACCESS.newExpression(v))) + Collection> markings = visibilities.stream().map(v -> AccessExpressionUtil.toAccessExpressionMarkings(ACCESS.newExpression(v))) .collect(Collectors.toSet()); - try { - return AccessExpressionUtil.combine(new MarkingFunctions.Default(), markings); - } catch (MarkingFunctions.Exception e) { - throw new IllegalArgumentException("Failed to combine visibilities", e); - } + return (new MarkingFunctions.Default()).combine(markings); } @Override diff --git a/warehouse/query-core/src/main/java/datawave/core/iterators/FieldIndexCountingIteratorPerVisibility.java b/warehouse/query-core/src/main/java/datawave/core/iterators/FieldIndexCountingIteratorPerVisibility.java index d0af39e1f5c..a66eaada8a3 100644 --- a/warehouse/query-core/src/main/java/datawave/core/iterators/FieldIndexCountingIteratorPerVisibility.java +++ b/warehouse/query-core/src/main/java/datawave/core/iterators/FieldIndexCountingIteratorPerVisibility.java @@ -567,11 +567,9 @@ private Map buildReturnKeys() { ColumnVisibility cv = null; try { // Calculate the columnVisibility for this key from the combiner. - cv = AccessExpressionUtil.combine(MarkingFunctions.Factory.createMarkingFunctions(), - columnVisibilities.stream() - .map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))) - .collect(Collectors.toSet())) - .toColumnVisibility(); + cv = (new MarkingFunctions.Default()).combine(columnVisibilities.stream() + .map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))) + .collect(Collectors.toSet())).toColumnVisibility(); } catch (Exception e) { log.error("Could not create combined columnVisibility for the count", e); return null; diff --git a/warehouse/query-core/src/main/java/datawave/core/iterators/GlobalIndexDateSummaryIterator.java b/warehouse/query-core/src/main/java/datawave/core/iterators/GlobalIndexDateSummaryIterator.java index 328a13c261f..80178336ddc 100644 --- a/warehouse/query-core/src/main/java/datawave/core/iterators/GlobalIndexDateSummaryIterator.java +++ b/warehouse/query-core/src/main/java/datawave/core/iterators/GlobalIndexDateSummaryIterator.java @@ -258,11 +258,9 @@ public Map getKeyValues() throws IOException { Set columnVisibilities = this.columnVisibilitiesMap.get(datatype); // Note that the access controls found in the combined ColumnVisibility will be pulled out appropriately here - ColumnVisibility cv = AccessExpressionUtil.combine(MarkingFunctions.Factory.createMarkingFunctions(), - columnVisibilities.stream() - .map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))) - .collect(Collectors.toSet())) - .toColumnVisibility(); + ColumnVisibility cv = (new MarkingFunctions.Default()).combine(columnVisibilities.stream() + .map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))) + .collect(Collectors.toSet())).toColumnVisibility(); // Create a new Key compatible with the shardIndex key format Key k = new Key(this.fieldValue, this.fieldName, this.date + '\0' + datatype, new String(cv.getExpression())); diff --git a/warehouse/query-core/src/main/java/datawave/core/iterators/ResultCountingIterator.java b/warehouse/query-core/src/main/java/datawave/core/iterators/ResultCountingIterator.java index 77901e4499d..ad837049419 100644 --- a/warehouse/query-core/src/main/java/datawave/core/iterators/ResultCountingIterator.java +++ b/warehouse/query-core/src/main/java/datawave/core/iterators/ResultCountingIterator.java @@ -210,14 +210,9 @@ public Value getTopValue() { ColumnVisibility cv = null; - try { - cv = AccessExpressionUtil.combine(MarkingFunctions.Factory.createMarkingFunctions(), columnVisibilities.stream() - .map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))).collect(Collectors.toSet())) - .toColumnVisibility(); - } catch (MarkingFunctions.Exception e) { - log.error("Could not create combined columnVisibility for the count", e); - return null; - } + cv = (new MarkingFunctions.Default()).combine(columnVisibilities.stream() + .map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))).collect(Collectors.toSet())) + .toColumnVisibility(); ResultCountTuple result = new ResultCountTuple(this.count, cv); ByteArrayOutputStream baos = new ByteArrayOutputStream(); diff --git a/warehouse/query-core/src/main/java/datawave/query/attributes/AttributeBag.java b/warehouse/query-core/src/main/java/datawave/query/attributes/AttributeBag.java index 597b99b18c5..6f73ef42877 100644 --- a/warehouse/query-core/src/main/java/datawave/query/attributes/AttributeBag.java +++ b/warehouse/query-core/src/main/java/datawave/query/attributes/AttributeBag.java @@ -23,7 +23,7 @@ public abstract class AttributeBag> extends Attribute private static final long ONE_DAY_MS = 1000l * 60 * 60 * 24; - public MarkingFunctions getMarkingFunctions() { + public MarkingFunctions getMarkingFunctions() { return MarkingFunctions.Factory.createMarkingFunctions(); } @@ -43,8 +43,8 @@ public void invalidateMetadata() { this.validMetadata = false; } - public boolean isValidMetadata() { - return (this.validMetadata && isMetadataSet()); + public boolean isNotValidMetadata() { + return (!this.validMetadata || !isMetadataSet()); } public abstract Collection>> getAttributes(); @@ -54,14 +54,14 @@ public boolean isValidMetadata() { @Override public long getTimestamp() { // calling isMetadataSet first to update the metadata as needed - if (isValidMetadata() == false) + if (isNotValidMetadata()) this.updateMetadata(); return super.getTimestamp(); } @Override public ColumnVisibility getColumnVisibility() { - if (isValidMetadata() == false) + if (isNotValidMetadata()) this.updateMetadata(); return super.getColumnVisibility(); } @@ -81,9 +81,8 @@ private void updateMetadata() { protected ColumnVisibility combineAndSetColumnVisibilities(Collection>> attributes) throws Exception { Collection accessExpressions = attributes.stream().map(attr -> AccessExpressionUtil.toAccessExpression(attr.getColumnVisibility())) .collect(Collectors.toSet()); - return AccessExpressionUtil - .combine(MarkingFunctions.Factory.createMarkingFunctions(), - accessExpressions.stream().map(AccessExpressionUtil::toAccessExpressionMarkings).collect(Collectors.toList())) + return (new MarkingFunctions.Default()) + .combine(accessExpressions.stream().map(AccessExpressionUtil::toAccessExpressionMarkings).collect(Collectors.toList())) .toColumnVisibility(); } diff --git a/warehouse/query-core/src/main/java/datawave/query/common/grouping/GroupingUtils.java b/warehouse/query-core/src/main/java/datawave/query/common/grouping/GroupingUtils.java index 42fea0584b3..f3e3daf1767 100644 --- a/warehouse/query-core/src/main/java/datawave/query/common/grouping/GroupingUtils.java +++ b/warehouse/query-core/src/main/java/datawave/query/common/grouping/GroupingUtils.java @@ -47,8 +47,7 @@ public enum AverageAggregatorWriteFormat { */ public static ColumnVisibility combineVisibilities(Collection expressions, MarkingFunctions markingFunctions, boolean failOnError) { try { - return AccessExpressionUtil - .combine(markingFunctions, expressions.stream().map(AccessExpressionUtil::toAccessExpressionMarkings).collect(Collectors.toList())) + return markingFunctions.combine(expressions.stream().map(AccessExpressionUtil::toAccessExpressionMarkings).collect(Collectors.toList())) .toColumnVisibility(); } catch (MarkingFunctions.Exception e) { if (failOnError) { diff --git a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java index fb95db77f9e..355c5d592c1 100644 --- a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java +++ b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java @@ -31,6 +31,7 @@ import com.google.protobuf.InvalidProtocolBufferException; import datawave.ingest.protobuf.Uid; +import datawave.marking.AccessExpressionMarkings; import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.query.Constants; @@ -38,7 +39,7 @@ public class DiscoveryIterator implements SortedKeyValueIterator { private static final Logger log = Logger.getLogger(DiscoveryIterator.class); - private static final MarkingFunctions markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); + private static final MarkingFunctions markingFunctions = new MarkingFunctions.Default(); private Key key; private Value value; @@ -163,11 +164,9 @@ private DiscoveredThing aggregate(Collection termEntries) { } else { // Otherwise, combine the visibilities, and return the aggregated result. try { - ColumnVisibility visibility = AccessExpressionUtil.combine(markingFunctions, - visibilities.stream() - .map(cv -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(cv))) - .collect(Collectors.toSet())) - .toColumnVisibility(); + ColumnVisibility visibility = markingFunctions.combine(visibilities.stream() + .map(cv -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(cv))) + .collect(Collectors.toSet())).toColumnVisibility(); MapWritable countsByVis = new MapWritable(); visibilityToCounts.forEach((key, value) -> countsByVis.put(new Text(key), new LongWritable(value))); String normalizedVis = AccessExpressionUtil.normalize(AccessExpressionUtil.toAccessExpression(visibility)).getExpression(); diff --git a/warehouse/query-core/src/main/java/datawave/query/discovery/TermInfoAggregation.java b/warehouse/query-core/src/main/java/datawave/query/discovery/TermInfoAggregation.java index d9138e3cdfe..50bd440b0e1 100644 --- a/warehouse/query-core/src/main/java/datawave/query/discovery/TermInfoAggregation.java +++ b/warehouse/query-core/src/main/java/datawave/query/discovery/TermInfoAggregation.java @@ -110,11 +110,9 @@ public DiscoveredThing apply(Collection from) { ColumnVisibility columnVisibility = null; try { - columnVisibility = AccessExpressionUtil.combine(MarkingFunctions.Factory.createMarkingFunctions(), - columnVisibilities.stream() - .map(cv -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(cv))) - .collect(Collectors.toSet())) - .toColumnVisibility(); + columnVisibility = (new MarkingFunctions.Default()).combine(columnVisibilities.stream() + .map(cv -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(cv))) + .collect(Collectors.toSet())).toColumnVisibility(); } catch (Exception e) { log.warn("Invalid columnvisibility after combining!", e); diff --git a/warehouse/query-core/src/main/java/datawave/query/iterators/FieldIndexCountingIterator.java b/warehouse/query-core/src/main/java/datawave/query/iterators/FieldIndexCountingIterator.java index 83f4ac5a741..be31fb3b4f4 100644 --- a/warehouse/query-core/src/main/java/datawave/query/iterators/FieldIndexCountingIterator.java +++ b/warehouse/query-core/src/main/java/datawave/query/iterators/FieldIndexCountingIterator.java @@ -588,14 +588,9 @@ private Key buildReturnKey() { columnVisibilities.add(new ColumnVisibility(t)); } ColumnVisibility cv; - try { - cv = AccessExpressionUtil.combine(MarkingFunctions.Factory.createMarkingFunctions(), columnVisibilities.stream() - .map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))).collect(Collectors.toSet())) - .toColumnVisibility(); - } catch (MarkingFunctions.Exception e) { - log.error("Could not combine visibilities: " + visibilitySet + " " + e); - return null; - } + cv = (new MarkingFunctions.Default()).combine(columnVisibilities.stream() + .map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))).collect(Collectors.toSet())) + .toColumnVisibility(); return new Key(this.currentRow, new Text(this.currentFieldName), cq, new Text(cv.getExpression()), this.maxTimeStamp); } diff --git a/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java b/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java index 810ebc8a5c5..b364839197e 100644 --- a/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java +++ b/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java @@ -47,7 +47,6 @@ import datawave.core.iterators.DatawaveFieldIndexListIteratorJexl; import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; -import datawave.marking.MarkingFunctionsFactory; import datawave.query.attributes.Attribute; import datawave.query.attributes.Attributes; import datawave.query.attributes.ValueTuple; @@ -470,15 +469,9 @@ private void addHitsForFunction(String field, ASTFunctionNode node, HitListArith } } - try { - cv = AccessExpressionUtil.combine(MarkingFunctionsFactory.createMarkingFunctions(), cvs.stream() - .map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))).collect(Collectors.toSet())) - .toColumnVisibility(); - } catch (MarkingFunctions.Exception e) { - log.error("Failed to combine column visibilities while generating HIT_TERM for phrase function for field [" + field + "]"); - log.error("msg: ", e); - return; - } + cv = (new MarkingFunctions.Default()).combine(cvs.stream() + .map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))).collect(Collectors.toSet())) + .toColumnVisibility(); source.setColumnVisibility(cv); // create an Attributes backed ValueTuple diff --git a/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java b/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java index 502f3e68ee3..fc9173d7a71 100644 --- a/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java +++ b/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java @@ -48,7 +48,6 @@ public class ValueToAttributes implements Function,Iterable> compositeToFieldMap; private Map> compositeFieldSeparatorsByType; - private final MarkingFunctions markingFunctions; private final Multimap> componentFieldToValues = ArrayListMultimap.create(); private final EventDataQueryFilter attrFilter; @@ -93,7 +92,6 @@ public ValueToAttributes(CompositeMetadata compositeMetadata, TypeMetadata typeM public ValueToAttributes(AttributeFactory attributeFactory, CompositeMetadata compositeMetadata, EventDataQueryFilter attrFilter, MarkingFunctions markingFunctions, boolean fromIndex) { this.attrFactory = attributeFactory; - this.markingFunctions = markingFunctions; this.attrFilter = attrFilter; if (compositeMetadata != null) { this.compositeToFieldMap = compositeMetadata.getCompositeFieldMapByType(); @@ -293,9 +291,8 @@ public Attribute joinAttributes(String compositeName, Collection if (log.isDebugEnabled()) { log.debug("dataList is " + dataList); } - ColumnVisibility combinedColumnVisibility = AccessExpressionUtil - .combine(this.markingFunctions, - accessExpressions.stream().map(AccessExpressionUtil::toAccessExpressionMarkings).collect(Collectors.toList())) + ColumnVisibility combinedColumnVisibility = (new MarkingFunctions.Default()) + .combine(accessExpressions.stream().map(AccessExpressionUtil::toAccessExpressionMarkings).collect(Collectors.toList())) .toColumnVisibility(); metadata = new Key(metadata.getRow(), metadata.getColumnFamily(), new Text(), combinedColumnVisibility, timestamp); if (dataList.size() == 1) { diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountAggregatingIterator.java b/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountAggregatingIterator.java index afc3facd7d5..82d2d33d154 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountAggregatingIterator.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountAggregatingIterator.java @@ -154,11 +154,9 @@ public Object getAggregatedEvent() { private ColumnVisibility getCombinedColumnVisibility() { try { - return AccessExpressionUtil.combine(markingFunctions, - columnVisibilities.stream() - .map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))) - .collect(Collectors.toSet())) - .toColumnVisibility(); + return (new MarkingFunctions.Default()).combine(columnVisibilities.stream() + .map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))) + .collect(Collectors.toSet())).toColumnVisibility(); } catch (Exception e) { log.error("Could not combine columnVisibilities for the count", e); return null; diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountResultPostprocessor.java b/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountResultPostprocessor.java index 6dc38992c2f..124a2618032 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountResultPostprocessor.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountResultPostprocessor.java @@ -72,11 +72,9 @@ public void apply(List results) { if (success) { ColumnVisibility columnVisibility = null; try { - columnVisibility = AccessExpressionUtil.combine(markingFunctions, - columnVisibilities.stream() - .map(cv -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(cv))) - .collect(Collectors.toSet())) - .toColumnVisibility(); + columnVisibility = (new MarkingFunctions.Default()).combine(columnVisibilities.stream() + .map(cv -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(cv))) + .collect(Collectors.toSet())).toColumnVisibility(); } catch (Exception e) { log.error("Could not create combined columnVisibilities for the count", e); } diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java index 56d2ea70bfa..f9c5adf0926 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java @@ -478,13 +478,13 @@ public Map buildSummary(Iterator iter, long maxValues) { public static class Tuple { - private final MarkingFunctions tupleMarkingFunctions; + private final MarkingFunctions markingFunctions; private long count = 0L; private long maxTimestamp = 0L; Set uniqueVisibilities = new HashSet<>(); public Tuple(MarkingFunctions markingFunctions) { - tupleMarkingFunctions = markingFunctions; + this.markingFunctions = markingFunctions; } public void aggregate(Key key, Value val) { @@ -504,21 +504,14 @@ public long getMaxTimestamp() { } public ColumnVisibility getColumnVisibility() { - try { - Set columnVisibilities = new HashSet<>(); - for (Text t : this.uniqueVisibilities) { - columnVisibilities.add(new ColumnVisibility(t)); - } - return AccessExpressionUtil.combine(tupleMarkingFunctions, - columnVisibilities.stream() - .map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))) - .collect(Collectors.toList())) - .toColumnVisibility(); - - } catch (MarkingFunctions.Exception e) { - logger.error("Could not create combined column visibility for the count", e); - return null; + Set columnVisibilities = new HashSet<>(); + for (Text t : this.uniqueVisibilities) { + columnVisibilities.add(new ColumnVisibility(t)); } + return (new MarkingFunctions.Default()).combine(columnVisibilities.stream() + .map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))).collect(Collectors.toList())) + .toColumnVisibility(); + } } diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformerSupport.java b/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformerSupport.java index 81704053f57..c637e9b6407 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformerSupport.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformerSupport.java @@ -18,7 +18,6 @@ import datawave.core.query.logic.BaseQueryLogicTransformer; import datawave.edge.model.EdgeModelFields; import datawave.edge.util.EdgeValue; -import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.microservice.query.Query; @@ -108,7 +107,7 @@ public BaseQueryResponse createResponse(List resultList) { response.addEdge(edge); } - response.setMarkings(AccessExpressionUtil.combine(this.markingFunctions, uniqueMarkings)); + response.setMarkings(markingFunctions.combine(uniqueMarkings)); return response; } catch (Exception ex) { throw new RuntimeException("could not handle markings in resultList ", ex); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java index 18be701e996..587ee33a6b1 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java @@ -16,9 +16,9 @@ import datawave.core.query.cachedresults.CacheableLogic; import datawave.core.query.logic.BaseQueryLogic; +import datawave.marking.AccessExpressionMarkings; import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; -import datawave.marking.MarkingFunctions.Exception; import datawave.marking.Markings; import datawave.microservice.query.Query; import datawave.query.Constants; @@ -84,7 +84,7 @@ public EventBase transform(Entry entry) { // Hold unique Markings and merge them at the end // for the overall event visibility. - Set> markingsToMerge = new HashSet<>(); + Collection> markingsToMerge = new HashSet<>(); for (Entry> e : eventFields.asMap().entrySet()) { origFieldName = e.getKey(); @@ -95,7 +95,8 @@ public EventBase transform(Entry entry) { } for (FieldValue fv : e.getValue()) { - Markings fieldMarkings = AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(fv.getVisibility())); + AccessExpressionMarkings fieldMarkings = AccessExpressionUtil + .toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(fv.getVisibility())); markingsToMerge.add(fieldMarkings); String value = new String(fv.getValue(), StandardCharsets.UTF_8); // if this is a content field name, then replace the value with the uid @@ -106,11 +107,7 @@ public EventBase transform(Entry entry) { } } - try { - event.setMarkings(AccessExpressionUtil.combine(this.markingFunctions, markingsToMerge)); - } catch (Exception e1) { - throw new RuntimeException("could not combine markings"); - } + event.setMarkings((new MarkingFunctions.Default()).combine(markingsToMerge)); event.setFields(new ArrayList<>(values)); Metadata metadata = new Metadata(); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java index 2505a335f66..54490282bc3 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java @@ -167,7 +167,7 @@ protected FacetsBase buildResponse(Document document, Key documentKey, ColumnVis @Override public BaseQueryResponse createResponse(List resultList) { FacetQueryResponseBase response = responseObjectFactory.getFacetQueryResponse(); - Set> combinedMarkings = new HashSet<>(); + Collection> combinedMarkings = new HashSet<>(); for (Object result : resultList) { FacetsBase facet = (FacetsBase) result; @@ -184,12 +184,7 @@ public BaseQueryResponse createResponse(List resultList) { } } - try { - response.setMarkings(AccessExpressionUtil.combine(this.markingFunctions, combinedMarkings)); - } catch (MarkingFunctions.Exception e) { - log.warn(e); - // original ignored these exceptions - } + response.setMarkings((new MarkingFunctions.Default()).combine(combinedMarkings)); return response; } diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/FieldIndexCountQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/FieldIndexCountQueryTransformer.java index 202cba48e0d..b50fcbbcffa 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/FieldIndexCountQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/FieldIndexCountQueryTransformer.java @@ -141,7 +141,7 @@ public BaseQueryResponse createResponse(List resultList) { response.setFields(variableFieldList); response.setEvents(eventList); response.setReturnedEvents((long) eventList.size()); - return (BaseQueryResponse) response; + return response; } @Override diff --git a/warehouse/query-core/src/test/java/datawave/query/predicate/ValueToAttributesTest.java b/warehouse/query-core/src/test/java/datawave/query/predicate/ValueToAttributesTest.java index 01eca42e877..c5f7deb9e2e 100644 --- a/warehouse/query-core/src/test/java/datawave/query/predicate/ValueToAttributesTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/predicate/ValueToAttributesTest.java @@ -36,6 +36,7 @@ import datawave.core.query.configuration.GenericQueryConfiguration; import datawave.helpers.PrintUtility; import datawave.ingest.data.TypeRegistry; +import datawave.marking.AccessExpressionMarkings; import datawave.marking.MarkingFunctions; import datawave.microservice.query.QueryImpl; import datawave.query.QueryTestTableHelper; @@ -260,7 +261,7 @@ public void testComposites() { TypeMetadata typeMetadata = new TypeMetadata( "dts:[0:beep];types:[0:datawave.data.type.DateType,1:datawave.data.type.IpAddressType,2:datawave.data.type.LcNoDiacriticsType,3:datawave.data.type.NoOpType,4:datawave.data.type.NumberType];MAKE:[0:2];MAKE_COLOR:[0:3];START_DATE:[0:0];TYPE_NOEVAL:[0:2];IP_ADDR:[0:1];WHEELS:[0:2,0:4];COLOR:[0:2];COLOR_WHEELS:[0:3];TYPE:[0:2]"); - MarkingFunctions markingFunctions = new MarkingFunctions.Default(); + MarkingFunctions markingFunctions = new MarkingFunctions.Default(); ValueToAttributes valueToAttributes = new ValueToAttributes(compositeMetadata, typeMetadata, null, markingFunctions, true); } } diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepIngestQueryTest.java b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepIngestQueryTest.java index b8cd3438d0d..bb679633a3e 100644 --- a/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepIngestQueryTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepIngestQueryTest.java @@ -29,6 +29,7 @@ import datawave.core.query.result.event.DefaultResponseObjectFactory; import datawave.helpers.PrintUtility; import datawave.ingest.mapreduce.handler.ssdeep.SSDeepIndexHandler; +import datawave.marking.AccessExpressionMarkings; import datawave.marking.MarkingFunctions; import datawave.microservice.query.QueryImpl; import datawave.microservice.querymetric.QueryMetricFactoryImpl; @@ -87,7 +88,7 @@ public static void filterSetup() throws Exception { @Before public void setupQuery() { - MarkingFunctions markingFunctions = new MarkingFunctions.Default(); + MarkingFunctions markingFunctions = new MarkingFunctions.Default(); ResponseObjectFactory responseFactory = new DefaultResponseObjectFactory(); MetadataHelperFactory metadataHelperFactory = new MetadataHelperFactory(); diff --git a/warehouse/query-core/src/test/java/datawave/query/transformer/annotation/AnnotationHitsTransformerTest.java b/warehouse/query-core/src/test/java/datawave/query/transformer/annotation/AnnotationHitsTransformerTest.java index 5ee5b2134b4..077bae65f68 100644 --- a/warehouse/query-core/src/test/java/datawave/query/transformer/annotation/AnnotationHitsTransformerTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/transformer/annotation/AnnotationHitsTransformerTest.java @@ -42,6 +42,7 @@ import datawave.annotation.protobuf.v1.SegmentBoundary; import datawave.annotation.protobuf.v1.SegmentValue; import datawave.data.normalizer.Normalizer; +import datawave.marking.AccessExpressionMarkings; import datawave.marking.MarkingFunctions; import datawave.microservice.query.Query; import datawave.microservice.query.QueryImpl; @@ -117,7 +118,7 @@ public class AnnotationHitsTransformerTest { private ShardQueryConfiguration shardQueryConfiguration; private Query settings; - private MarkingFunctions markingFunctions; + private MarkingFunctions markingFunctions; private List annotations; private AnnotationSource annotationSource; From f8687df858b26ff8aece814454fda20e6e39d966 Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Tue, 21 Apr 2026 12:49:29 -0400 Subject: [PATCH 24/63] rebase fix --- .../modification/MutableMetadataHandlerTestSupport.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/modification/src/test/java/datawave/modification/MutableMetadataHandlerTestSupport.java b/core/modification/src/test/java/datawave/modification/MutableMetadataHandlerTestSupport.java index 298954850c9..916c8564f98 100644 --- a/core/modification/src/test/java/datawave/modification/MutableMetadataHandlerTestSupport.java +++ b/core/modification/src/test/java/datawave/modification/MutableMetadataHandlerTestSupport.java @@ -126,7 +126,7 @@ DefaultModificationRequest newMarkingsInsertRequest(String fieldName, String fie request.setFieldName(fieldName); request.setFieldValue(fieldValue); Map markings = new HashMap<>(); - markings.put(MarkingFunctions.Default.COLUMN_VISIBILITY, columnVisibility); + markings.put("columnVisibility", columnVisibility); request.setFieldMarkings(markings); request.setColumnVisibility(null); return request; From 9ab4f0953b55630cc484341da832abca4b682be9 Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Thu, 23 Apr 2026 15:44:56 -0400 Subject: [PATCH 25/63] restore flatten functions in mapreduce handler --- .../handler/dateindex/DateIndexDataTypeHandler.java | 8 ++++---- .../handler/edge/ProtobufEdgeDataTypeHandler.java | 2 +- .../mapreduce/handler/error/ErrorDataTypeHandler.java | 3 +-- .../ingest/mapreduce/handler/facet/FacetHandler.java | 5 +---- .../mapreduce/handler/shard/ShardedDataTypeHandler.java | 3 +-- 5 files changed, 8 insertions(+), 13 deletions(-) diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/dateindex/DateIndexDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/dateindex/DateIndexDataTypeHandler.java index 545fe5da54c..bdd02f57535 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/dateindex/DateIndexDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/dateindex/DateIndexDataTypeHandler.java @@ -1,7 +1,5 @@ package datawave.ingest.mapreduce.handler.dateindex; -import static java.nio.charset.StandardCharsets.UTF_8; - import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; @@ -39,7 +37,7 @@ import datawave.ingest.mapreduce.job.BulkIngestKey; import datawave.ingest.metadata.RawRecordMetadata; import datawave.ingest.table.aggregator.DateIndexDateAggregator; -import datawave.marking.AccessExpressionUtil; +import datawave.marking.MarkingFunctions; /** *

@@ -96,6 +94,7 @@ public class DateIndexDataTypeHandler implements DataTypeHandler, protected DateNormalizer dateNormalizer = new DateNormalizer(); protected ShardIdFactory shardIdFactory = null; protected TaskAttemptContext taskAttemptContext = null; + protected MarkingFunctions markingFunctions; public Set getDataTypes() { Set types = new HashSet<>(); @@ -150,6 +149,7 @@ public void setup(TaskAttemptContext context) { this.taskAttemptContext = context; this.conf = context.getConfiguration(); this.shardIdFactory = new ShardIdFactory(conf); + markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); String tableName = conf.get(DATEINDEX_TNAME, null); if (null == tableName) { @@ -366,7 +366,7 @@ private Value createDateIndexValue(int shard) { * @return the flattened visibility */ protected byte[] flatten(ColumnVisibility vis) { - return AccessExpressionUtil.normalize(AccessExpressionUtil.toAccessExpression(vis)).getExpression().getBytes(UTF_8); + return markingFunctions == null ? vis.flatten() : markingFunctions.flatten(vis); } public Text getDateIndexTableName() { diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDataTypeHandler.java index 8f3f24bb86c..4638824ae8b 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDataTypeHandler.java @@ -1366,7 +1366,7 @@ protected Text getDurationVisibility(EdgeDataBundle value) { * @return the flattened visibility */ protected byte[] flatten(ColumnVisibility vis) { - return AccessExpressionUtil.normalize(AccessExpressionUtil.toAccessExpression(vis)).getExpression().getBytes(UTF_8); + return markingFunctions == null ? vis.flatten() : markingFunctions.flatten(vis); } @Override diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorDataTypeHandler.java index f791cab06b1..9ca84e117e1 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorDataTypeHandler.java @@ -36,7 +36,6 @@ import datawave.ingest.mapreduce.job.BulkIngestKey; import datawave.ingest.mapreduce.job.writer.ContextWriter; import datawave.ingest.metadata.RawRecordMetadata; -import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.util.TextUtil; import datawave.util.time.DateHelper; @@ -335,7 +334,7 @@ protected byte[] getVisibility(RawRecordContainer event, NormalizedContentInterf * @return the normalized visibility bytes */ protected byte[] flatten(ColumnVisibility vis) { - return AccessExpressionUtil.normalize(AccessExpressionUtil.toAccessExpression(vis)).getExpression().getBytes(UTF_8); + return markingFunctions == null ? vis.flatten() : markingFunctions.flatten(vis); } @Override diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/facet/FacetHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/facet/FacetHandler.java index d205e84f6b2..d72497337ee 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/facet/FacetHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/facet/FacetHandler.java @@ -1,7 +1,5 @@ package datawave.ingest.mapreduce.handler.facet; -import static java.nio.charset.StandardCharsets.UTF_8; - import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; @@ -47,7 +45,6 @@ import datawave.ingest.mapreduce.job.BulkIngestKey; import datawave.ingest.mapreduce.job.writer.ContextWriter; import datawave.ingest.metadata.RawRecordMetadata; -import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.util.time.DateHelper; @@ -195,7 +192,7 @@ public RawRecordMetadata getMetadata() { } protected byte[] flatten(ColumnVisibility vis) { - return AccessExpressionUtil.normalize(AccessExpressionUtil.toAccessExpression(vis)).getExpression().getBytes(UTF_8); + return markingFunctions == null ? vis.flatten() : markingFunctions.flatten(vis); } @Override diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java index c770302d20b..6d46ed335aa 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java @@ -51,7 +51,6 @@ import datawave.ingest.util.BloomFilterUtil; import datawave.ingest.util.BloomFilterWrapper; import datawave.ingest.util.DiskSpaceStarvationStrategy; -import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.query.model.Direction; @@ -1148,7 +1147,7 @@ public byte[] getVisibility(RawRecordContainer event, NormalizedContentInterface * @return the normalized visibility bytes */ protected byte[] flatten(ColumnVisibility vis) { - return AccessExpressionUtil.normalize(AccessExpressionUtil.toAccessExpression(vis)).getExpression().getBytes(UTF_8); + return markingFunctions == null ? vis.flatten() : markingFunctions.flatten(vis); } /** From 2b812e812b698387abede9a539f3dd05875ca69e Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Tue, 28 Apr 2026 15:56:23 -0400 Subject: [PATCH 26/63] bugfixes --- .../src/main/java/datawave/marking/MarkingFunctions.java | 5 ++++- .../java/datawave/query/attributes/AttributeBag.java | 9 ++------- .../query/transformer/DocumentTransformerSupport.java | 6 +----- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java index adeaed2d5c1..e773765d0c3 100644 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java @@ -72,7 +72,10 @@ class Default implements MarkingFunctions { @Override public ColumnVisibility combineVisibilities(Collection visibilities) { - throw new UnsupportedOperationException(); + return AccessExpressionUtil.toColumnVisibility(combine( + visibilities.stream().map(vis -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(vis))) + .collect(Collectors.toList())) + .toAccessExpression()); } @Override diff --git a/warehouse/query-core/src/main/java/datawave/query/attributes/AttributeBag.java b/warehouse/query-core/src/main/java/datawave/query/attributes/AttributeBag.java index 6f73ef42877..c61cd41f6b0 100644 --- a/warehouse/query-core/src/main/java/datawave/query/attributes/AttributeBag.java +++ b/warehouse/query-core/src/main/java/datawave/query/attributes/AttributeBag.java @@ -4,13 +4,11 @@ import java.util.Collection; import java.util.stream.Collectors; -import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.commons.lang.mutable.MutableLong; import org.apache.log4j.Logger; -import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.MarkingFunctions.Exception; @@ -79,11 +77,8 @@ private void updateMetadata() { } protected ColumnVisibility combineAndSetColumnVisibilities(Collection>> attributes) throws Exception { - Collection accessExpressions = attributes.stream().map(attr -> AccessExpressionUtil.toAccessExpression(attr.getColumnVisibility())) - .collect(Collectors.toSet()); - return (new MarkingFunctions.Default()) - .combine(accessExpressions.stream().map(AccessExpressionUtil::toAccessExpressionMarkings).collect(Collectors.toList())) - .toColumnVisibility(); + Collection visibilities = attributes.stream().map(Attribute::getColumnVisibility).collect(Collectors.toList()); + return (getMarkingFunctions()).combineVisibilities(visibilities); } private long updateTimestamps() { diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformerSupport.java b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformerSupport.java index bbed63d923f..a0cac35e236 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformerSupport.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformerSupport.java @@ -1,7 +1,5 @@ package datawave.query.transformer; -import static datawave.marking.AccessExpressionMarkings.ACCESS; - import java.io.File; import java.io.IOException; import java.text.SimpleDateFormat; @@ -30,7 +28,6 @@ import datawave.core.query.logic.BaseQueryLogic; import datawave.core.query.logic.WritesQueryMetrics; import datawave.core.query.logic.WritesResultCardinalities; -import datawave.marking.AccessExpressionMarkings; import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; @@ -73,7 +70,6 @@ public abstract class DocumentTransformerSupport extends EventQueryTransfor protected Boolean reducedResponse; private static final Logger log = Logger.getLogger(DocumentTransformerSupport.class); - private static final Markings EMPTY_MARKINGS = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression("")).build(); private long sourceCount = 0; private long nextCount = 0; @@ -476,7 +472,7 @@ protected Collection> buildDocumentFields(Key documentKey, String f } } else { // noinspection RedundantCast - FieldBase f = createField(fieldName, (Long) null, attr, EMPTY_MARKINGS, (String) null); + FieldBase f = createField(fieldName, (Long) null, attr, null, (String) null); myFields.add(f); } } From 7916bce4f55c697d7049de7fec760388bf2393a0 Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Tue, 28 Apr 2026 15:56:28 -0400 Subject: [PATCH 27/63] remove EMPTY_MARKINGS --- .../core/query/cachedresults/CacheableQueryRowImpl.java | 8 +------- .../src/main/java/datawave/marking/MarkingFunctions.java | 6 ++---- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java b/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java index aeedbe831b5..ebf22311ffb 100644 --- a/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java +++ b/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java @@ -1,7 +1,5 @@ package datawave.core.query.cachedresults; -import static datawave.marking.AccessExpressionMarkings.ACCESS; - import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -18,7 +16,6 @@ import com.google.common.collect.Sets; import datawave.data.type.Type; -import datawave.marking.AccessExpressionMarkings; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.webservice.query.cachedresults.CacheableQueryRow; @@ -28,7 +25,6 @@ public class CacheableQueryRowImpl extends CacheableQueryRow implements ObjectSizeOf { private static final Logger LOGGER = LoggerFactory.getLogger(CacheableQueryRowImpl.class); - private static final Markings EMPTY_MARKINGS = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression("")).build(); private String user = null; private String queryId = null; @@ -37,7 +33,7 @@ public class CacheableQueryRowImpl extends CacheableQueryRow implements ObjectSi private String eventId = null; private String row = null; private String colFam = null; - private Markings markings = EMPTY_MARKINGS; + private Markings markings = null; private Map> columnMarkingsMap = new HashMap<>(); private Map columnColumnVisibilityMap = new HashMap<>(); private Map columnTypeMap = new HashMap<>(); @@ -183,8 +179,6 @@ public Markings getColumnMarkings(String columnName) { markings = this.columnMarkingsMap.get("_DEFAULT_"); } - if (markings == null) - markings = EMPTY_MARKINGS; return markings; } diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java index e773765d0c3..11371792aa6 100644 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java @@ -68,8 +68,6 @@ public Exception(Throwable cause) { class Default implements MarkingFunctions { - private static final AccessExpressionMarkings EMPTY_MARKINGS = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression("")).build(); - @Override public ColumnVisibility combineVisibilities(Collection visibilities) { return AccessExpressionUtil.toColumnVisibility(combine( @@ -91,7 +89,7 @@ public AccessExpressionMarkings combine(Collection> markings) { } if (uniqueMarkings.isEmpty()) { - return EMPTY_MARKINGS; + return null; } if (uniqueMarkings.size() == 1) { @@ -131,7 +129,7 @@ public AccessExpressionMarkings combine(Markings markings1, Markings marki } } } - return markings1 == null ? EMPTY_MARKINGS : (AccessExpressionMarkings) markings1; + return markings1 == null ? null : (AccessExpressionMarkings) markings1; } @Override From 745c734927170b15a54f360b4fc5d140b2a60576 Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Thu, 30 Apr 2026 11:52:01 -0400 Subject: [PATCH 28/63] marking combine bugfixes --- .../marking/AccessExpressionUtil.java | 6 +++++ .../datawave/marking/MarkingFunctions.java | 11 +++++--- .../FrequencyMetadataAggregator.java | 11 +++----- .../metadata/MetadataDescriptionsHelper.java | 10 +++++-- .../querymetric/QueryMetricOperations.java | 5 +++- ...eldIndexCountingIteratorPerVisibility.java | 12 ++++----- .../GlobalIndexDateSummaryIterator.java | 11 ++++---- .../iterators/ResultCountingIterator.java | 21 +++++++++------ .../query/attributes/AttributeBag.java | 9 +++++-- .../datawave/query/attributes/Attributes.java | 5 ---- .../datawave/query/attributes/Document.java | 4 --- .../query/discovery/DiscoveryIterator.java | 11 ++++---- .../query/discovery/TermInfoAggregation.java | 14 +++++----- .../iterators/FieldIndexCountingIterator.java | 20 +++++++++----- .../query/jexl/DatawaveInterpreter.java | 25 +++++++++++------ .../query/predicate/ValueToAttributes.java | 27 ++++++++++++------- .../shard/CountAggregatingIterator.java | 11 ++++---- .../shard/CountResultPostprocessor.java | 11 ++++---- .../shard/FieldIndexCountQueryLogic.java | 16 ++++++----- 19 files changed, 141 insertions(+), 99 deletions(-) diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionUtil.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionUtil.java index 55464281d8e..850e207b588 100644 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionUtil.java +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionUtil.java @@ -35,6 +35,9 @@ private AccessExpressionUtil() {} * @return the equivalent column visibility */ public static ColumnVisibility toColumnVisibility(AccessExpression ae) { + if (null == ae) { + return null; + } String expr = ae.getExpression(); if (expr.isEmpty()) { return EMPTY_VISIBILITY; @@ -50,6 +53,9 @@ public static ColumnVisibility toColumnVisibility(AccessExpression ae) { * @return the equivalent access expression */ public static AccessExpression toAccessExpression(ColumnVisibility cv) { + if (null == cv) { + return null; + } byte[] expr = cv.getExpression(); if (expr.length == 0) { return EMPTY_EXPRESSION; diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java index 11371792aa6..d2865e2bc83 100644 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java @@ -70,10 +70,15 @@ class Default implements MarkingFunctions { @Override public ColumnVisibility combineVisibilities(Collection visibilities) { - return AccessExpressionUtil.toColumnVisibility(combine( + AccessExpressionMarkings accessExpressionMarkings = combine( visibilities.stream().map(vis -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(vis))) - .collect(Collectors.toList())) - .toAccessExpression()); + .collect(Collectors.toList())); + + if (null == accessExpressionMarkings) { + return null; + } + + return AccessExpressionUtil.toColumnVisibility(accessExpressionMarkings.toAccessExpression()); } @Override diff --git a/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java b/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java index 3e07a1622db..52469c132e7 100644 --- a/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java +++ b/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java @@ -10,7 +10,6 @@ import java.util.Map; import java.util.Set; import java.util.TreeMap; -import java.util.stream.Collectors; import org.apache.accumulo.core.data.ByteSequence; import org.apache.accumulo.core.data.Key; @@ -31,8 +30,6 @@ import com.google.common.base.Splitter; import datawave.iterator.ReducingIterator; -import datawave.marking.AccessExpressionMarkings; -import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.query.model.DateFrequencyMap; import datawave.util.StringUtils; @@ -57,7 +54,7 @@ public class FrequencyMetadataAggregator extends WrappingIterator implements Opt private static final Logger log = Logger.getLogger(FrequencyMetadataAggregator.class); private static final String NULL_BYTE = "\0"; - private static final MarkingFunctions markingFunctions = new MarkingFunctions.Default(); + private final MarkingFunctions markingFunctions; private boolean combineVisibilities; private String columnsOption; @@ -85,6 +82,7 @@ public FrequencyMetadataAggregator() { cache = new TreeMap<>(); visibilityToDateFrequencies = new HashMap<>(); visibilityToMaxTimestamp = new HashMap<>(); + markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); } @Override @@ -587,10 +585,7 @@ private Map buildCacheEntries() { private ColumnVisibility combineAllVisibilities() { Set visibilities = visibilityToDateFrequencies.keySet(); try { - return markingFunctions.combine( - visibilities.stream().map(cv -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(cv))) - .collect(Collectors.toSet())) - .toColumnVisibility(); + return markingFunctions.combineVisibilities(visibilities); } catch (MarkingFunctions.Exception e) { throw new RuntimeException("Could not combine markings", e); } diff --git a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java index 1293e12d354..cd793186a47 100644 --- a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java +++ b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java @@ -124,7 +124,10 @@ public void setDescriptions(MetadataEntry entry, Set bw = accumuloClient.createBatchWriter(metadataTableName, bwConfig); Mutation m = new Mutation(entry.getFieldName()); for (DescriptionBase desc : descs) { - ColumnVisibility cv = desc.getMarkings().toColumnVisibility(); + ColumnVisibility cv = null; + if (desc.getMarkings() != null) { + cv = desc.getMarkings().toColumnVisibility(); + } m.put(ColumnFamilyConstants.COLF_DESC, new Text(entry.getDatatype()), cv, new Value(desc.getDescription().getBytes(UTF_8))); } bw.addMutation(m); @@ -157,7 +160,10 @@ public void removeDescription(MetadataEntry entry, DescriptionBase desc) throws BatchWriterConfig bwConfig = new BatchWriterConfig().setMaxMemory(10000L).setMaxLatency(100L, TimeUnit.MILLISECONDS).setMaxWriteThreads(1); bw = accumuloClient.createBatchWriter(metadataTableName, bwConfig); Mutation m = new Mutation(entry.getFieldName()); - ColumnVisibility cv = desc.getMarkings().toColumnVisibility(); + ColumnVisibility cv = null; + if (desc.getMarkings() != null) { + cv = desc.getMarkings().toColumnVisibility(); + } m.putDelete(ColumnFamilyConstants.COLF_DESC, new Text(entry.getDatatype()), cv); bw.addMutation(m); } finally { diff --git a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java index 15529a03013..79ee9eafb16 100644 --- a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java +++ b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java @@ -478,7 +478,10 @@ private boolean isMetricsAdministrator(DatawaveUserDetails currentUser) { private boolean canViewMetric(DatawaveUserDetails currentUser, BaseQueryMetric metric) throws Exception { boolean userCanViewMetric = true; - ColumnVisibility columnVisibility = metric.getMarkings().toColumnVisibility(); + ColumnVisibility columnVisibility = null; + if (null != metric.getMarkings()) { + columnVisibility = metric.getMarkings().toColumnVisibility(); + } for (DatawaveUser user : currentUser.getProxiedUsers()) { Authorizations authorizations = new Authorizations(user.getAuths().toArray(new String[0])); VisibilityEvaluator visibilityEvaluator = new VisibilityEvaluator(authorizations); diff --git a/warehouse/query-core/src/main/java/datawave/core/iterators/FieldIndexCountingIteratorPerVisibility.java b/warehouse/query-core/src/main/java/datawave/core/iterators/FieldIndexCountingIteratorPerVisibility.java index a66eaada8a3..4a532d353b0 100644 --- a/warehouse/query-core/src/main/java/datawave/core/iterators/FieldIndexCountingIteratorPerVisibility.java +++ b/warehouse/query-core/src/main/java/datawave/core/iterators/FieldIndexCountingIteratorPerVisibility.java @@ -12,7 +12,6 @@ import java.util.TimeZone; import java.util.TreeMap; import java.util.TreeSet; -import java.util.stream.Collectors; import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.data.ArrayByteSequence; @@ -35,7 +34,6 @@ import com.google.common.collect.Sets; import datawave.iterators.IteratorSettingHelper; -import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.query.Constants; import datawave.util.CompositeTimestamp; @@ -100,6 +98,7 @@ public class FieldIndexCountingIteratorPerVisibility extends WrappingIterator im private Set columnVisibilities = Sets.newHashSet(); private TreeMap keyCache = null; + private MarkingFunctions markingFunctions; // ------------------------------------------------------------------------- // ------------- Constructors @@ -564,12 +563,13 @@ private Map buildReturnKeys() { } } - ColumnVisibility cv = null; + ColumnVisibility cv; try { + if (null == markingFunctions) { + markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); + } // Calculate the columnVisibility for this key from the combiner. - cv = (new MarkingFunctions.Default()).combine(columnVisibilities.stream() - .map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))) - .collect(Collectors.toSet())).toColumnVisibility(); + cv = markingFunctions.combineVisibilities(columnVisibilities); } catch (Exception e) { log.error("Could not create combined columnVisibility for the count", e); return null; diff --git a/warehouse/query-core/src/main/java/datawave/core/iterators/GlobalIndexDateSummaryIterator.java b/warehouse/query-core/src/main/java/datawave/core/iterators/GlobalIndexDateSummaryIterator.java index 80178336ddc..83be5504f39 100644 --- a/warehouse/query-core/src/main/java/datawave/core/iterators/GlobalIndexDateSummaryIterator.java +++ b/warehouse/query-core/src/main/java/datawave/core/iterators/GlobalIndexDateSummaryIterator.java @@ -8,7 +8,6 @@ import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; -import java.util.stream.Collectors; import org.apache.accumulo.core.data.ByteSequence; import org.apache.accumulo.core.data.Key; @@ -28,7 +27,6 @@ import datawave.ingest.protobuf.Uid; import datawave.ingest.protobuf.Uid.List.Builder; -import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.query.Constants; @@ -203,6 +201,7 @@ protected static class TermInfoSummary { private String date = null; private Map summary = new HashMap<>(); private Map> columnVisibilitiesMap = Maps.newHashMap(); + protected MarkingFunctions markingFunctions; public TermInfoSummary(String fieldValue, String fieldName, String date) { this.fieldValue = fieldValue; @@ -257,12 +256,14 @@ public Map getKeyValues() throws IOException { // Calculate the ColumnVisibility for this key from the combiner. Set columnVisibilities = this.columnVisibilitiesMap.get(datatype); + if (null == markingFunctions) { + markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); + } // Note that the access controls found in the combined ColumnVisibility will be pulled out appropriately here - ColumnVisibility cv = (new MarkingFunctions.Default()).combine(columnVisibilities.stream() - .map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))) - .collect(Collectors.toSet())).toColumnVisibility(); + ColumnVisibility cv = markingFunctions.combineVisibilities(columnVisibilities); // Create a new Key compatible with the shardIndex key format + assert cv != null; Key k = new Key(this.fieldValue, this.fieldName, this.date + '\0' + datatype, new String(cv.getExpression())); // Create a UID object with just the count for the Value diff --git a/warehouse/query-core/src/main/java/datawave/core/iterators/ResultCountingIterator.java b/warehouse/query-core/src/main/java/datawave/core/iterators/ResultCountingIterator.java index ad837049419..6a746a9fa3d 100644 --- a/warehouse/query-core/src/main/java/datawave/core/iterators/ResultCountingIterator.java +++ b/warehouse/query-core/src/main/java/datawave/core/iterators/ResultCountingIterator.java @@ -7,7 +7,6 @@ import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; import org.apache.accumulo.core.client.BatchScanner; import org.apache.accumulo.core.data.ByteSequence; @@ -32,7 +31,6 @@ import com.google.common.cache.CacheBuilder; import com.google.common.collect.Sets; -import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; /** @@ -68,6 +66,7 @@ public long read() { private String threadName = null; protected Set columnVisibilities = Sets.newHashSet(); + protected MarkingFunctions markingFunctions; public ResultCountingIterator() { threadName = Thread.currentThread().getName(); @@ -208,13 +207,19 @@ public Value getTopValue() { sw.start(); - ColumnVisibility cv = null; - - cv = (new MarkingFunctions.Default()).combine(columnVisibilities.stream() - .map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))).collect(Collectors.toSet())) - .toColumnVisibility(); + if (null == markingFunctions) { + markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); + } + ColumnVisibility columnVisibility; + try { + columnVisibility = markingFunctions.combineVisibilities(columnVisibilities); + } catch (MarkingFunctions.Exception e) { + String message = "Could not create combined ColumnVisibility"; + log.error(message, e); + throw new RuntimeException(message, e); + } - ResultCountTuple result = new ResultCountTuple(this.count, cv); + ResultCountTuple result = new ResultCountTuple(this.count, columnVisibility); ByteArrayOutputStream baos = new ByteArrayOutputStream(); Output kryoOutput = new Output(baos); kryo.writeObject(kryoOutput, result); diff --git a/warehouse/query-core/src/main/java/datawave/query/attributes/AttributeBag.java b/warehouse/query-core/src/main/java/datawave/query/attributes/AttributeBag.java index c61cd41f6b0..fe9f73a14bf 100644 --- a/warehouse/query-core/src/main/java/datawave/query/attributes/AttributeBag.java +++ b/warehouse/query-core/src/main/java/datawave/query/attributes/AttributeBag.java @@ -18,11 +18,15 @@ public abstract class AttributeBag> extends Attribute private static final Logger log = Logger.getLogger(AttributeBag.class); protected long shardTimestamp = Long.MAX_VALUE; protected boolean validMetadata = false; + private MarkingFunctions markingFunctions; private static final long ONE_DAY_MS = 1000l * 60 * 60 * 24; public MarkingFunctions getMarkingFunctions() { - return MarkingFunctions.Factory.createMarkingFunctions(); + if (null == markingFunctions) { + markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); + } + return markingFunctions; } protected AttributeBag() { @@ -78,7 +82,8 @@ private void updateMetadata() { protected ColumnVisibility combineAndSetColumnVisibilities(Collection>> attributes) throws Exception { Collection visibilities = attributes.stream().map(Attribute::getColumnVisibility).collect(Collectors.toList()); - return (getMarkingFunctions()).combineVisibilities(visibilities); + markingFunctions = getMarkingFunctions(); + return markingFunctions.combineVisibilities(visibilities); } private long updateTimestamps() { diff --git a/warehouse/query-core/src/main/java/datawave/query/attributes/Attributes.java b/warehouse/query-core/src/main/java/datawave/query/attributes/Attributes.java index 7a83a35ad62..4bf62e1a1bf 100644 --- a/warehouse/query-core/src/main/java/datawave/query/attributes/Attributes.java +++ b/warehouse/query-core/src/main/java/datawave/query/attributes/Attributes.java @@ -20,7 +20,6 @@ import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; -import datawave.marking.MarkingFunctions; import datawave.query.collections.FunctionalSet; import datawave.query.jexl.DatawaveJexlContext; @@ -38,10 +37,6 @@ public class Attributes extends AttributeBag implements Serializable */ private boolean trackSizes; - public MarkingFunctions getMarkingFunctions() { - return MarkingFunctions.Factory.createMarkingFunctions(); - } - protected Attributes() { this(true); } diff --git a/warehouse/query-core/src/main/java/datawave/query/attributes/Document.java b/warehouse/query-core/src/main/java/datawave/query/attributes/Document.java index b28d57538af..6d45802bb32 100644 --- a/warehouse/query-core/src/main/java/datawave/query/attributes/Document.java +++ b/warehouse/query-core/src/main/java/datawave/query/attributes/Document.java @@ -82,10 +82,6 @@ public Long load(Text row) { private static final long ONE_DAY_MS = 1000L * 60 * 60 * 24; - public MarkingFunctions getMarkingFunctions() { - return MarkingFunctions.Factory.createMarkingFunctions(); - } - public Markings getMarkings() { return AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(getColumnVisibility())); } diff --git a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java index 355c5d592c1..7cc11510513 100644 --- a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java +++ b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java @@ -31,7 +31,6 @@ import com.google.protobuf.InvalidProtocolBufferException; import datawave.ingest.protobuf.Uid; -import datawave.marking.AccessExpressionMarkings; import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.query.Constants; @@ -39,7 +38,6 @@ public class DiscoveryIterator implements SortedKeyValueIterator { private static final Logger log = Logger.getLogger(DiscoveryIterator.class); - private static final MarkingFunctions markingFunctions = new MarkingFunctions.Default(); private Key key; private Value value; @@ -48,6 +46,7 @@ public class DiscoveryIterator implements SortedKeyValueIterator { private boolean showReferenceCount = false; private boolean reverseIndex = false; private boolean sumCounts = false; + private MarkingFunctions markingFunctions; @Override public DiscoveryIterator deepCopy(IteratorEnvironment env) { @@ -164,9 +163,11 @@ private DiscoveredThing aggregate(Collection termEntries) { } else { // Otherwise, combine the visibilities, and return the aggregated result. try { - ColumnVisibility visibility = markingFunctions.combine(visibilities.stream() - .map(cv -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(cv))) - .collect(Collectors.toSet())).toColumnVisibility(); + if (null == markingFunctions) { + markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); + } + ColumnVisibility visibility = markingFunctions.combineVisibilities(visibilities); + MapWritable countsByVis = new MapWritable(); visibilityToCounts.forEach((key, value) -> countsByVis.put(new Text(key), new LongWritable(value))); String normalizedVis = AccessExpressionUtil.normalize(AccessExpressionUtil.toAccessExpression(visibility)).getExpression(); diff --git a/warehouse/query-core/src/main/java/datawave/query/discovery/TermInfoAggregation.java b/warehouse/query-core/src/main/java/datawave/query/discovery/TermInfoAggregation.java index 50bd440b0e1..0adec8c8334 100644 --- a/warehouse/query-core/src/main/java/datawave/query/discovery/TermInfoAggregation.java +++ b/warehouse/query-core/src/main/java/datawave/query/discovery/TermInfoAggregation.java @@ -5,7 +5,6 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; -import java.util.stream.Collectors; import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.hadoop.io.MapWritable; @@ -26,6 +25,7 @@ public class TermInfoAggregation implements Function,Discov private final boolean separateCountsByColumnVisibility; private boolean showReferenceCountInsteadOfTermCount = false; private boolean reverseIndex = false; + private MarkingFunctions markingFunctions; public TermInfoAggregation() { this.separateCountsByColumnVisibility = false; @@ -107,13 +107,12 @@ public DiscoveredThing apply(Collection from) { log.trace("Did not aggregate any counts for [" + term + "][" + field + "][" + type + "][" + date + "]. Returning null."); return null; } else { - ColumnVisibility columnVisibility = null; + ColumnVisibility columnVisibility; try { - - columnVisibility = (new MarkingFunctions.Default()).combine(columnVisibilities.stream() - .map(cv -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(cv))) - .collect(Collectors.toSet())).toColumnVisibility(); - + if (null == markingFunctions) { + markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); + } + columnVisibility = markingFunctions.combineVisibilities(columnVisibilities); } catch (Exception e) { log.warn("Invalid columnvisibility after combining!", e); return null; @@ -124,6 +123,7 @@ public DiscoveredThing apply(Collection from) { countsByVis.put(new Text(entry.getKey()), new VLongWritable(entry.getValue())); } + assert columnVisibility != null; return new DiscoveredThing(term, field, type, date, AccessExpressionUtil.normalize(AccessExpressionUtil.toAccessExpression(columnVisibility)).getExpression(), count, countsByVis); } diff --git a/warehouse/query-core/src/main/java/datawave/query/iterators/FieldIndexCountingIterator.java b/warehouse/query-core/src/main/java/datawave/query/iterators/FieldIndexCountingIterator.java index be31fb3b4f4..761967052f3 100644 --- a/warehouse/query-core/src/main/java/datawave/query/iterators/FieldIndexCountingIterator.java +++ b/warehouse/query-core/src/main/java/datawave/query/iterators/FieldIndexCountingIterator.java @@ -11,7 +11,6 @@ import java.util.Set; import java.util.TimeZone; import java.util.TreeSet; -import java.util.stream.Collectors; import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.data.ArrayByteSequence; @@ -32,7 +31,6 @@ import datawave.core.common.util.TypeFilter; import datawave.iterators.IteratorSettingHelper; -import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.query.Constants; import datawave.util.CompositeTimestamp; @@ -93,6 +91,7 @@ public class FieldIndexCountingIterator extends WrappingIterator implements Sort public static final Text fi_PREFIX_TEXT = new Text("fi\u0000"); private Set visibilitySet = new HashSet<>(); + private MarkingFunctions markingFunctions; // ------------------------------------------------------------------------- // ------------- Constructors @@ -587,12 +586,19 @@ private Key buildReturnKey() { for (Text t : this.visibilitySet) { columnVisibilities.add(new ColumnVisibility(t)); } - ColumnVisibility cv; - cv = (new MarkingFunctions.Default()).combine(columnVisibilities.stream() - .map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))).collect(Collectors.toSet())) - .toColumnVisibility(); + ColumnVisibility columnVisibility; + try { + if (null == markingFunctions) { + markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); + } + columnVisibility = markingFunctions.combineVisibilities(columnVisibilities); + } catch (Exception e) { + log.warn("Invalid columnvisibility after combining!", e); + return null; + } - return new Key(this.currentRow, new Text(this.currentFieldName), cq, new Text(cv.getExpression()), this.maxTimeStamp); + assert columnVisibility != null; + return new Key(this.currentRow, new Text(this.currentFieldName), cq, new Text(columnVisibility.getExpression()), this.maxTimeStamp); } /* TODO: make this a mutable long, also check wrap up current method */ diff --git a/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java b/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java index b364839197e..af2a495b866 100644 --- a/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java +++ b/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java @@ -15,7 +15,6 @@ import java.util.Map; import java.util.Set; import java.util.SortedSet; -import java.util.stream.Collectors; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.security.ColumnVisibility; @@ -45,7 +44,6 @@ import com.google.common.collect.Maps; import datawave.core.iterators.DatawaveFieldIndexListIteratorJexl; -import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.query.attributes.Attribute; import datawave.query.attributes.Attributes; @@ -72,6 +70,7 @@ public class DatawaveInterpreter extends Interpreter { protected Map resultMap; + private MarkingFunctions markingFunctions; private static final Logger log = Logger.getLogger(DatawaveInterpreter.class); @@ -453,7 +452,7 @@ private void addHitsForFunction(Object o, ASTFunctionNode node) { private void addHitsForFunction(String field, ASTFunctionNode node, HitListArithmetic hitListArithmetic) { ColumnVisibility cv; // aggregate individual hits for the content function - Collection cvs = new HashSet<>(); + Collection columnVisibilities = new HashSet<>(); Attributes source = new Attributes(true); ContentFunctionsDescriptor.ContentJexlArgumentDescriptor jexlArgDescriptor = new ContentFunctionsDescriptor().getArgumentDescriptor(node); @@ -464,15 +463,25 @@ private void addHitsForFunction(String field, ASTFunctionNode node, HitListArith if (values.contains(tuple.getNormalizedValue())) { Attribute attr = tuple.getSource(); source.add(attr); - cvs.add(attr.getColumnVisibility()); + columnVisibilities.add(attr.getColumnVisibility()); } } } - cv = (new MarkingFunctions.Default()).combine(cvs.stream() - .map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))).collect(Collectors.toSet())) - .toColumnVisibility(); - source.setColumnVisibility(cv); + if (null == markingFunctions) { + markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); + } + ColumnVisibility columnVisibility; + try { + columnVisibility = markingFunctions.combineVisibilities(columnVisibilities); + } catch (MarkingFunctions.Exception e) { + String message = "Could not create combined ColumnVisibility"; + log.error(message, e); + throw new RuntimeException(message, e); + } + + assert columnVisibility != null; + source.setColumnVisibility(columnVisibility); // create an Attributes backed ValueTuple String phrase = jexlArgDescriptor.getHitTermValue(); diff --git a/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java b/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java index fc9173d7a71..36cda86b4f5 100644 --- a/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java +++ b/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java @@ -7,9 +7,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.stream.Collectors; -import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.data.ByteSequence; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.security.ColumnVisibility; @@ -25,7 +23,6 @@ import datawave.data.type.OneToManyNormalizerType; import datawave.data.type.Type; import datawave.ingest.data.config.ingest.CompositeIngest; -import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.MarkingFunctions.Exception; import datawave.query.attributes.Attribute; @@ -54,6 +51,7 @@ public class ValueToAttributes implements Function,Iterable markingFunctions; /** * Constructor that accepts raw components @@ -226,7 +224,7 @@ public Attribute getFieldValue(String fieldName, Key k) { } public Attribute joinAttributes(String compositeName, Collection> in, boolean isOverloadedComposite, String separator) throws Exception { - Collection accessExpressions = new HashSet<>(); + Collection columnVisibilities = new HashSet<>(); List dataList = new ArrayList<>(); long timestamp = 0; boolean toKeep = false; @@ -261,7 +259,7 @@ public Attribute joinAttributes(String compositeName, Collection dataList.add(base + value); } timestamp = Math.max(timestamp, attrs.getTimestamp()); - accessExpressions.add(AccessExpressionUtil.toAccessExpression(attrs.getColumnVisibility())); + columnVisibilities.add(attrs.getColumnVisibility()); } } } @@ -285,16 +283,25 @@ public Attribute joinAttributes(String compositeName, Collection } } timestamp = Math.max(timestamp, attr.getTimestamp()); - accessExpressions.add(AccessExpressionUtil.toAccessExpression(attr.getColumnVisibility())); + columnVisibilities.add(attr.getColumnVisibility()); } } if (log.isDebugEnabled()) { log.debug("dataList is " + dataList); } - ColumnVisibility combinedColumnVisibility = (new MarkingFunctions.Default()) - .combine(accessExpressions.stream().map(AccessExpressionUtil::toAccessExpressionMarkings).collect(Collectors.toList())) - .toColumnVisibility(); - metadata = new Key(metadata.getRow(), metadata.getColumnFamily(), new Text(), combinedColumnVisibility, timestamp); + ColumnVisibility columnVisibility; + try { + if (null == markingFunctions) { + markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); + } + columnVisibility = markingFunctions.combineVisibilities(columnVisibilities); + } catch (java.lang.Exception e) { + log.warn("Invalid columnvisibility after combining!", e); + return null; + } + + assert columnVisibility != null; + metadata = new Key(metadata.getRow(), metadata.getColumnFamily(), new Text(), columnVisibility, timestamp); if (dataList.size() == 1) { return this.attrFactory.create(compositeName, dataList.get(0), metadata, toKeep, true); } else { diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountAggregatingIterator.java b/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountAggregatingIterator.java index 82d2d33d154..110637c8f92 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountAggregatingIterator.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountAggregatingIterator.java @@ -13,7 +13,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; -import java.util.stream.Collectors; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; @@ -26,7 +25,6 @@ import com.esotericsoftware.kryo.io.Input; import com.google.common.collect.Maps; -import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.webservice.query.result.event.DefaultEvent; @@ -127,7 +125,7 @@ private static class CountEntryAggregator { private final Set columnVisibilities = new HashSet<>(); private final Transformer transformer; - private final MarkingFunctions markingFunctions; + private MarkingFunctions markingFunctions; public CountEntryAggregator(Transformer transformer, MarkingFunctions markingFunctions) { this.transformer = transformer; @@ -154,9 +152,10 @@ public Object getAggregatedEvent() { private ColumnVisibility getCombinedColumnVisibility() { try { - return (new MarkingFunctions.Default()).combine(columnVisibilities.stream() - .map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))) - .collect(Collectors.toSet())).toColumnVisibility(); + if (null == markingFunctions) { + markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); + } + return markingFunctions.combineVisibilities(columnVisibilities); } catch (Exception e) { log.error("Could not combine columnVisibilities for the count", e); return null; diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountResultPostprocessor.java b/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountResultPostprocessor.java index 124a2618032..d093279ec55 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountResultPostprocessor.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountResultPostprocessor.java @@ -5,7 +5,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Set; -import java.util.stream.Collectors; import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.log4j.Logger; @@ -13,7 +12,6 @@ import com.google.common.collect.Sets; import datawave.core.query.logic.ResultPostprocessor; -import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.webservice.query.result.event.EventBase; import datawave.webservice.query.result.event.FieldBase; @@ -21,7 +19,7 @@ public class CountResultPostprocessor implements ResultPostprocessor { private static final Logger log = Logger.getLogger(CountResultPostprocessor.class); - private final MarkingFunctions markingFunctions; + private MarkingFunctions markingFunctions; public CountResultPostprocessor(MarkingFunctions markingFunctions) { this.markingFunctions = markingFunctions; @@ -72,9 +70,10 @@ public void apply(List results) { if (success) { ColumnVisibility columnVisibility = null; try { - columnVisibility = (new MarkingFunctions.Default()).combine(columnVisibilities.stream() - .map(cv -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(cv))) - .collect(Collectors.toSet())).toColumnVisibility(); + if (null == markingFunctions) { + markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); + } + columnVisibility = markingFunctions.combineVisibilities(columnVisibilities); } catch (Exception e) { log.error("Could not create combined columnVisibilities for the count", e); } diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java index f9c5adf0926..91623cf9b74 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java @@ -14,7 +14,6 @@ import java.util.Map.Entry; import java.util.Set; import java.util.TreeSet; -import java.util.stream.Collectors; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.BatchScanner; @@ -33,7 +32,6 @@ import datawave.core.query.configuration.GenericQueryConfiguration; import datawave.core.query.logic.QueryLogicTransformer; import datawave.data.type.Type; -import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.microservice.query.Query; import datawave.query.Constants; @@ -478,7 +476,7 @@ public Map buildSummary(Iterator iter, long maxValues) { public static class Tuple { - private final MarkingFunctions markingFunctions; + private MarkingFunctions markingFunctions; private long count = 0L; private long maxTimestamp = 0L; Set uniqueVisibilities = new HashSet<>(); @@ -508,9 +506,15 @@ public ColumnVisibility getColumnVisibility() { for (Text t : this.uniqueVisibilities) { columnVisibilities.add(new ColumnVisibility(t)); } - return (new MarkingFunctions.Default()).combine(columnVisibilities.stream() - .map(c -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(c))).collect(Collectors.toList())) - .toColumnVisibility(); + try { + if (null == markingFunctions) { + markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); + } + return markingFunctions.combineVisibilities(columnVisibilities); + } catch (Exception e) { + log.warn("Invalid columnvisibility after combining!", e); + return null; + } } } From 04401a58e058bb6224496ea1431e7b82feb8f0e0 Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Thu, 30 Apr 2026 12:01:55 -0400 Subject: [PATCH 29/63] rebase fixes --- microservices/services/accumulo/api/pom.xml | 2 +- microservices/services/dictionary/api/pom.xml | 2 +- .../services/modification/api/pom.xml | 2 +- microservices/services/query/api/pom.xml | 2 +- .../datawave/query/common/grouping/Group.java | 24 ++++++++++++ .../query/common/grouping/GroupingUtils.java | 3 +- .../common/grouping/DocumentGrouperTest.java | 39 ++++++++----------- .../test/java/datawave/test/GroupAssert.java | 20 +++++----- 8 files changed, 55 insertions(+), 39 deletions(-) diff --git a/microservices/services/accumulo/api/pom.xml b/microservices/services/accumulo/api/pom.xml index ea9aa40fc54..2cc21dbca49 100644 --- a/microservices/services/accumulo/api/pom.xml +++ b/microservices/services/accumulo/api/pom.xml @@ -19,7 +19,7 @@ http://webservice.datawave.nsa/v1 1.17.1 - 7.38.0-SNAPSHOT + 7.39.0-SNAPSHOT 2.14.3 2.3.3 diff --git a/microservices/services/dictionary/api/pom.xml b/microservices/services/dictionary/api/pom.xml index 4244cc91c95..3e9d3723bdd 100644 --- a/microservices/services/dictionary/api/pom.xml +++ b/microservices/services/dictionary/api/pom.xml @@ -19,7 +19,7 @@ http://webservice.datawave.nsa/v1 3.20.0 - 7.38.0-SNAPSHOT + 7.39.0-SNAPSHOT 31.1-jre 2.3.3 1.6.2 diff --git a/microservices/services/modification/api/pom.xml b/microservices/services/modification/api/pom.xml index eeaf6ad657d..21f560fb6e0 100644 --- a/microservices/services/modification/api/pom.xml +++ b/microservices/services/modification/api/pom.xml @@ -18,7 +18,7 @@ 2.1.4-5792fed3-bulkv2 - 7.38.0-SNAPSHOT + 7.39.0-SNAPSHOT 31.1-jre diff --git a/microservices/services/query/api/pom.xml b/microservices/services/query/api/pom.xml index 49e2241a85e..28cc310bb35 100644 --- a/microservices/services/query/api/pom.xml +++ b/microservices/services/query/api/pom.xml @@ -20,7 +20,7 @@ http://webservice.datawave.nsa/v1 4.3 3.20.0 - 7.38.0-SNAPSHOT + 7.39.0-SNAPSHOT 31.1-jre 2.14.3 diff --git a/warehouse/query-core/src/main/java/datawave/query/common/grouping/Group.java b/warehouse/query-core/src/main/java/datawave/query/common/grouping/Group.java index 9e028551d2f..1173042745b 100644 --- a/warehouse/query-core/src/main/java/datawave/query/common/grouping/Group.java +++ b/warehouse/query-core/src/main/java/datawave/query/common/grouping/Group.java @@ -3,13 +3,17 @@ import java.util.Collection; import java.util.HashSet; import java.util.Set; +import java.util.stream.Collectors; import org.apache.accumulo.access.AccessExpression; +import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.commons.lang.builder.ToStringBuilder; import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; +import datawave.marking.AccessExpressionUtil; + /** * Represents a grouping of values for fields specified via the #GROUP_BY functionality, with information about the total number of times the grouping was seen, * values for target aggregation fields that were matched to this group, and the different column visibilities seen. @@ -81,6 +85,17 @@ public void addAttributeExpressions(Grouping grouping) { } } + /** + * Return the set of visibilities seen for the given attribute. + * + * @param attribute + * the attribute + * @return the access expressions converted to ColumnVisibilities seen for the given attribute + */ + public Collection getColumnVisibilitiesForAttribute(GroupingAttribute attribute) { + return attributeExpressions.get(attribute).stream().map(AccessExpressionUtil::toColumnVisibility).collect(Collectors.toSet()); + } + /** * Return the set of access expressions seen for the given attribute. * @@ -111,6 +126,15 @@ public Set getDocumentExpressions() { return documentExpressions; } + /** + * Return the set of all distinct visibilities from documents that we have seen this group in. + * + * @return the document access expressions converted into ColumnVisibilities + */ + public Set getDocumentVisibilities() { + return documentExpressions.stream().map(AccessExpressionUtil::toColumnVisibility).collect(Collectors.toSet()); + } + /** * Increment the number of times we have seen this grouping by one. */ diff --git a/warehouse/query-core/src/main/java/datawave/query/common/grouping/GroupingUtils.java b/warehouse/query-core/src/main/java/datawave/query/common/grouping/GroupingUtils.java index f3e3daf1767..676a26efcb3 100644 --- a/warehouse/query-core/src/main/java/datawave/query/common/grouping/GroupingUtils.java +++ b/warehouse/query-core/src/main/java/datawave/query/common/grouping/GroupingUtils.java @@ -47,8 +47,7 @@ public enum AverageAggregatorWriteFormat { */ public static ColumnVisibility combineVisibilities(Collection expressions, MarkingFunctions markingFunctions, boolean failOnError) { try { - return markingFunctions.combine(expressions.stream().map(AccessExpressionUtil::toAccessExpressionMarkings).collect(Collectors.toList())) - .toColumnVisibility(); + return markingFunctions.combineVisibilities(expressions.stream().map(AccessExpressionUtil::toColumnVisibility).collect(Collectors.toList())); } catch (MarkingFunctions.Exception e) { if (failOnError) { throw new IllegalArgumentException("Unable to combine access expressions: " + expressions, e); diff --git a/warehouse/query-core/src/test/java/datawave/query/common/grouping/DocumentGrouperTest.java b/warehouse/query-core/src/test/java/datawave/query/common/grouping/DocumentGrouperTest.java index 2526531d200..4e213de1e5f 100644 --- a/warehouse/query-core/src/test/java/datawave/query/common/grouping/DocumentGrouperTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/common/grouping/DocumentGrouperTest.java @@ -1,7 +1,5 @@ package datawave.query.common.grouping; -import static datawave.marking.AccessExpressionMarkings.ACCESS; - import java.math.BigDecimal; import java.util.AbstractMap; import java.util.ArrayList; @@ -11,7 +9,6 @@ import java.util.Map; import java.util.Set; -import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.security.ColumnVisibility; import org.assertj.core.util.Sets; @@ -41,10 +38,6 @@ public class DocumentGrouperTest { private static final ColumnVisibility COLVIS_I = new ColumnVisibility("I"); private static final ColumnVisibility COLVIS_ALL_E_I = new ColumnVisibility("ALL&E&I"); - private static final AccessExpression AE_ALL = ACCESS.newExpression("ALL"); - private static final AccessExpression AE_E = ACCESS.newExpression("E"); - private static final AccessExpression AE_I = ACCESS.newExpression("I"); - private static final AccessExpression AE_ALL_E_I = ACCESS.newExpression("ALL&E&I"); private static final Key key = new Key("test_key"); private static final Multimap inverseReverseMap = HashMultimap.create(); private static final Map reverseMap = new HashMap<>(); @@ -534,10 +527,10 @@ public void testGroupingBySingleFieldAcrossMultipleDocuments() { GroupsAssert groupsAssert = GroupsAssert.assertThat(groups); groupsAssert.hasTotalGroups(2); - groupsAssert.assertGroup(textKey("GENDER", "MALE")).hasCount(4).hasDocumentExpressions(AE_ALL, AE_I) - .hasAccessExpressionsForKey(textKey("GENDER", "MALE"), AE_ALL, AE_I); - groupsAssert.assertGroup(textKey("GENDER", "FEMALE")).hasCount(3).hasDocumentExpressions(AE_I, AE_E) - .hasAccessExpressionsForKey(textKey("GENDER", "FEMALE"), AE_I, AE_E); + groupsAssert.assertGroup(textKey("GENDER", "MALE")).hasCount(4).hasDocumentVisibilities(COLVIS_ALL, COLVIS_I) + .hasVisibilitiesForKey(textKey("GENDER", "MALE"), COLVIS_ALL, COLVIS_I); + groupsAssert.assertGroup(textKey("GENDER", "FEMALE")).hasCount(3).hasDocumentVisibilities(COLVIS_I, COLVIS_E) + .hasVisibilitiesForKey(textKey("GENDER", "FEMALE"), COLVIS_I, COLVIS_E); } @Test @@ -621,18 +614,18 @@ public void testGroupingByMultipleFieldsWithDifferentFormatsAcrossMultipleDocume GroupsAssert groupsAssert = GroupsAssert.assertThat(groups); groupsAssert.hasTotalGroups(7); - groupsAssert.assertGroup(textKey("GENDER", "MALE"), textKey("BUILDING", "West")).hasCount(1).hasDocumentExpressions(AE_ALL_E_I) - .hasAccessExpressionsForKey(textKey("GENDER", "MALE"), AE_ALL).hasAccessExpressionsForKey(textKey("BUILDING", "West"), AE_E); - groupsAssert.assertGroup(textKey("GENDER", "MALE"), textKey("BUILDING", "East")).hasCount(1).hasDocumentExpressions(AE_ALL_E_I) - .hasAccessExpressionsForKey(textKey("GENDER", "MALE"), AE_ALL).hasAccessExpressionsForKey(textKey("BUILDING", "East"), AE_I); - groupsAssert.assertGroup(textKey("GENDER", "FEMALE"), textKey("BUILDING", "West")).hasCount(1).hasDocumentExpressions(AE_ALL_E_I) - .hasAccessExpressionsForKey(textKey("GENDER", "FEMALE"), AE_ALL).hasAccessExpressionsForKey(textKey("BUILDING", "West"), AE_E); - groupsAssert.assertGroup(textKey("GENDER", "FEMALE"), textKey("BUILDING", "East")).hasCount(1).hasDocumentExpressions(AE_ALL_E_I) - .hasAccessExpressionsForKey(textKey("GENDER", "FEMALE"), AE_ALL).hasAccessExpressionsForKey(textKey("BUILDING", "East"), AE_I); - groupsAssert.assertGroup(textKey("GENDER", "MALE"), textKey("BUILDING", "Center")).hasCount(2).hasDocumentExpressions(AE_ALL) - .hasAccessExpressionsForKey(textKey("GENDER", "MALE"), AE_ALL).hasAccessExpressionsForKey(textKey("BUILDING", "Center"), AE_ALL); - groupsAssert.assertGroup(textKey("BUILDING", "North")).hasCount(1).hasDocumentExpressions(AE_ALL); - groupsAssert.assertGroup(textKey("BUILDING", "South")).hasCount(1).hasDocumentExpressions(AE_ALL); + groupsAssert.assertGroup(textKey("GENDER", "MALE"), textKey("BUILDING", "West")).hasCount(1).hasDocumentVisibilities(COLVIS_ALL_E_I) + .hasVisibilitiesForKey(textKey("GENDER", "MALE"), COLVIS_ALL).hasVisibilitiesForKey(textKey("BUILDING", "West"), COLVIS_E); + groupsAssert.assertGroup(textKey("GENDER", "MALE"), textKey("BUILDING", "East")).hasCount(1).hasDocumentVisibilities(COLVIS_ALL_E_I) + .hasVisibilitiesForKey(textKey("GENDER", "MALE"), COLVIS_ALL).hasVisibilitiesForKey(textKey("BUILDING", "East"), COLVIS_I); + groupsAssert.assertGroup(textKey("GENDER", "FEMALE"), textKey("BUILDING", "West")).hasCount(1).hasDocumentVisibilities(COLVIS_ALL_E_I) + .hasVisibilitiesForKey(textKey("GENDER", "FEMALE"), COLVIS_ALL).hasVisibilitiesForKey(textKey("BUILDING", "West"), COLVIS_E); + groupsAssert.assertGroup(textKey("GENDER", "FEMALE"), textKey("BUILDING", "East")).hasCount(1).hasDocumentVisibilities(COLVIS_ALL_E_I) + .hasVisibilitiesForKey(textKey("GENDER", "FEMALE"), COLVIS_ALL).hasVisibilitiesForKey(textKey("BUILDING", "East"), COLVIS_I); + groupsAssert.assertGroup(textKey("GENDER", "MALE"), textKey("BUILDING", "Center")).hasCount(2).hasDocumentVisibilities(COLVIS_ALL) + .hasVisibilitiesForKey(textKey("GENDER", "MALE"), COLVIS_ALL).hasVisibilitiesForKey(textKey("BUILDING", "Center"), COLVIS_ALL); + groupsAssert.assertGroup(textKey("BUILDING", "North")).hasCount(1).hasDocumentVisibilities(COLVIS_ALL); + groupsAssert.assertGroup(textKey("BUILDING", "South")).hasCount(1).hasDocumentVisibilities(COLVIS_ALL); } @Test diff --git a/warehouse/query-core/src/test/java/datawave/test/GroupAssert.java b/warehouse/query-core/src/test/java/datawave/test/GroupAssert.java index c8dc6e3d0c8..e90693f070d 100644 --- a/warehouse/query-core/src/test/java/datawave/test/GroupAssert.java +++ b/warehouse/query-core/src/test/java/datawave/test/GroupAssert.java @@ -5,7 +5,7 @@ import java.util.Collection; import java.util.Objects; -import org.apache.accumulo.access.AccessExpression; +import org.apache.accumulo.core.security.ColumnVisibility; import org.assertj.core.api.AbstractAssert; import org.assertj.core.api.Assertions; @@ -34,20 +34,20 @@ public GroupAssert hasCount(int count) { return this; } - public GroupAssert hasAccessExpressionsForKey(GroupingAttribute key, AccessExpression... expressions) { + public GroupAssert hasVisibilitiesForKey(GroupingAttribute key, ColumnVisibility... visibilities) { isNotNull(); - Collection actualExpressions = actual.getAccessExpressionsForAttribute(key); - Assertions.assertThat(actualExpressions).isNotNull().withFailMessage("Expected access expressions for %s to contain exactly %s but was %s", key, - Arrays.toString(expressions), actualExpressions).containsExactlyInAnyOrder(expressions); + Collection actualVisibilities = actual.getColumnVisibilitiesForAttribute(key); + Assertions.assertThat(actualVisibilities).isNotNull().withFailMessage("Expected access expressions for %s to contain exactly %s but was %s", key, + Arrays.toString(visibilities), actualVisibilities).containsExactlyInAnyOrder(visibilities); return this; } - public GroupAssert hasDocumentExpressions(AccessExpression... expressions) { + public GroupAssert hasDocumentVisibilities(ColumnVisibility... visibilities) { isNotNull(); - Collection actualExpressions = actual.getDocumentExpressions(); - Assertions.assertThat(actualExpressions).isNotNull() - .withFailMessage("Expected document expressions to contain exactly %s but was %s", Arrays.toString(expressions), actualExpressions) - .containsExactlyInAnyOrder(expressions); + Collection actualVisibilities = actual.getDocumentVisibilities(); + Assertions.assertThat(actualVisibilities).isNotNull() + .withFailMessage("Expected document expressions to contain exactly %s but was %s", Arrays.toString(visibilities), actualVisibilities) + .containsExactlyInAnyOrder(visibilities); return this; } From 256e8cf830e046c84539fde9e8d5d816948eac9e Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Thu, 30 Apr 2026 13:47:46 -0400 Subject: [PATCH 30/63] fix MarkingFunctions to return empty ColumnVisibility when no access expression markings are present --- .../src/main/java/datawave/marking/MarkingFunctions.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java index d2865e2bc83..556e009e96b 100644 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java @@ -75,7 +75,7 @@ public ColumnVisibility combineVisibilities(Collection visibil .collect(Collectors.toList())); if (null == accessExpressionMarkings) { - return null; + return new ColumnVisibility(); } return AccessExpressionUtil.toColumnVisibility(accessExpressionMarkings.toAccessExpression()); From 57267bd800dcf452e1e7117f8aa65146f19ba17a Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Thu, 30 Apr 2026 16:46:57 -0400 Subject: [PATCH 31/63] DoucmentTransformer bugfix --- .../datawave/query/transformer/DocumentTransformer.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java index c978a3edfa6..2331b11a278 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java @@ -16,7 +16,6 @@ import datawave.core.query.logic.Flushable; import datawave.core.query.logic.WritesQueryMetrics; import datawave.core.query.logic.WritesResultCardinalities; -import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.microservice.query.Query; @@ -166,7 +165,12 @@ private EventBase _transform(Entry documentEntry) throws EmptyObje protected EventBase buildResponse(Document document, Key documentKey, ColumnVisibility eventCV, String colf, String row, MarkingFunctions markingFunctions) { - Markings markings = AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(eventCV)); + Markings markings; + try { + markings = markingFunctions.translateFromColumnVisibility(eventCV); + } catch (MarkingFunctions.Exception e) { + throw new RuntimeException(e); + } EventBase event = null; final Collection> documentFields = buildDocumentFields(documentKey, null, document, eventCV, markingFunctions); From deead2b30a499b463eb6291e7ae8c967613e3fd6 Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Fri, 1 May 2026 13:40:57 -0400 Subject: [PATCH 32/63] more bugfixes --- .../main/java/datawave/query/attributes/AttributeBag.java | 2 +- .../src/main/java/datawave/query/attributes/Document.java | 8 ++++++-- .../query/transformer/DocumentTransformerSupport.java | 3 +-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/warehouse/query-core/src/main/java/datawave/query/attributes/AttributeBag.java b/warehouse/query-core/src/main/java/datawave/query/attributes/AttributeBag.java index fe9f73a14bf..abde462a435 100644 --- a/warehouse/query-core/src/main/java/datawave/query/attributes/AttributeBag.java +++ b/warehouse/query-core/src/main/java/datawave/query/attributes/AttributeBag.java @@ -18,7 +18,7 @@ public abstract class AttributeBag> extends Attribute private static final Logger log = Logger.getLogger(AttributeBag.class); protected long shardTimestamp = Long.MAX_VALUE; protected boolean validMetadata = false; - private MarkingFunctions markingFunctions; + protected MarkingFunctions markingFunctions; private static final long ONE_DAY_MS = 1000l * 60 * 60 * 24; diff --git a/warehouse/query-core/src/main/java/datawave/query/attributes/Document.java b/warehouse/query-core/src/main/java/datawave/query/attributes/Document.java index 6d45802bb32..c17aec1f0ad 100644 --- a/warehouse/query-core/src/main/java/datawave/query/attributes/Document.java +++ b/warehouse/query-core/src/main/java/datawave/query/attributes/Document.java @@ -33,7 +33,6 @@ import com.google.common.collect.Iterators; import com.google.common.collect.Sets; -import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.query.Constants; @@ -83,7 +82,12 @@ public Long load(Text row) { private static final long ONE_DAY_MS = 1000L * 60 * 60 * 24; public Markings getMarkings() { - return AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(getColumnVisibility())); + try { + markingFunctions = getMarkingFunctions(); + return markingFunctions.translateFromColumnVisibility(getColumnVisibility()); + } catch (MarkingFunctions.Exception ignored) { + return null; + } } public Document() { diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformerSupport.java b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformerSupport.java index a0cac35e236..4b10d82d684 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformerSupport.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformerSupport.java @@ -28,7 +28,6 @@ import datawave.core.query.logic.BaseQueryLogic; import datawave.core.query.logic.WritesQueryMetrics; import datawave.core.query.logic.WritesResultCardinalities; -import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.microservice.query.Query; @@ -464,7 +463,7 @@ protected Collection> buildDocumentFields(Key documentKey, String f // Use the markings on the Field if we're returning the markings to the client if (!this.reducedResponse) { try { - Markings markings = AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(attr.getColumnVisibility())); + Markings markings = markingFunctions.translateFromColumnVisibility(attr.getColumnVisibility()); FieldBase field = this.makeField(fieldName, markings, attr.getColumnVisibility(), attr.getTimestamp(), attr.getData()); myFields.add(field); } catch (Exception ex) { From 717ce375d3642be621d84c364fd08895df7e3efb Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Mon, 4 May 2026 16:44:45 -0400 Subject: [PATCH 33/63] fix CacheableQueryRowImpl addColumn --- .../cachedresults/CacheableQueryRowImpl.java | 52 +++++++++++++------ 1 file changed, 35 insertions(+), 17 deletions(-) diff --git a/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java b/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java index ebf22311ffb..55d420c082e 100644 --- a/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java +++ b/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java @@ -9,6 +9,7 @@ import java.util.Set; import java.util.TreeSet; +import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,23 +52,40 @@ public void addColumn(String columnName, TypedValue columnTypedValue, Markings columnVisibilities = Sets.newHashSet(); + columnVisibilities.add(markingFunctions.translateToColumnVisibility(this.markings)); + columnVisibilities.add(markingFunctions.translateToColumnVisibility(markings)); + ColumnVisibility combinedVisibility = markingFunctions.combineVisibilities(columnVisibilities); + + // use combined marking as new markings + this.markings = markingFunctions.translateFromColumnVisibility(combinedVisibility); + } catch (MarkingFunctions.Exception e) { + LOGGER.error("Invalid markings {} skipping column {} = {}", markings, columnName, columnTypedValue, e); + return; + } + } } } } From bf03060627d5644dee577e880c075ecd98fa38fc Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Thu, 7 May 2026 09:04:13 -0400 Subject: [PATCH 34/63] cleanup AccessExpressionUtils and MarkingFunctions.Default usage --- .../CacheableQueryRowReader.java | 12 ++--- .../modification/MutableMetadataHandler.java | 24 ++++++---- .../marking/AccessExpressionMarkings.java | 35 ++++++++++++++- .../marking/AccessExpressionUtil.java | 11 ----- .../datawave/marking/MarkingFunctions.java | 6 +-- .../accumulo/admin/AdminService.java | 5 +-- .../accumulo/lookup/LookupService.java | 6 +-- .../metadata/DefaultMetadataFieldScanner.java | 10 +++-- .../metadata/MetadataDescriptionsHelper.java | 7 ++- .../handler/ShardTableQueryMetricHandler.java | 3 +- .../util/keyword/DefaultTagCloudUtils.java | 23 ++++++---- .../java/datawave/util/keyword/TagCloud.java | 11 ++--- .../datawave/util/keyword/TagCloudUtils.java | 6 +-- .../resources/MarkingFunctionsContext.xml | 24 ++++++++++ .../query/discovery/DiscoveryTransformer.java | 20 ++++++--- .../query/discovery/TermInfoAggregation.java | 2 +- .../table/parser/EventKeyValueFactory.java | 6 +-- .../parser/TermFrequencyKeyValueFactory.java | 7 ++- .../transformer/EdgeQueryTransformer.java | 6 +-- .../transformer/EventQueryTransformer.java | 45 +++++++++++++------ .../query/transformer/FacetedTransformer.java | 35 ++++++++++++--- .../FieldIndexCountQueryTransformer.java | 8 +++- .../GroupingDocumentTransformer.java | 5 +-- .../ShardIndexQueryTransformer.java | 8 +++- .../ShardQueryCountTableTransformer.java | 11 +++-- .../transformer/TagCloudTransformer.java | 8 +++- 26 files changed, 228 insertions(+), 116 deletions(-) create mode 100644 warehouse/keyword-common/src/test/resources/MarkingFunctionsContext.xml diff --git a/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowReader.java b/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowReader.java index d7d27746774..7e32ef467ff 100644 --- a/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowReader.java +++ b/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowReader.java @@ -1,7 +1,5 @@ package datawave.core.query.cachedresults; -import static datawave.marking.AccessExpressionMarkings.ACCESS; - import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; @@ -11,10 +9,10 @@ import java.util.Set; import java.util.TreeSet; +import org.apache.accumulo.core.security.ColumnVisibility; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.webservice.query.cachedresults.CacheableQueryRow; @@ -92,7 +90,7 @@ public static CacheableQueryRow createRow(ResultSet cachedRowSet, Set fi if (columnToIndexMap.get("_markings_") != null) { String mStr = cachedRowSet.getString(columnToIndexMap.get("_markings_")); if (mStr != null && !mStr.isEmpty()) { - cqfc.setMarkings(AccessExpressionUtil.toAccessExpressionMarkings(ACCESS.newExpression(mStr))); + cqfc.setMarkings(markingFunctions.translateFromColumnVisibility(new ColumnVisibility(mStr))); } } if (columnToIndexMap.get("_column_markings_") != null) { @@ -106,10 +104,10 @@ public static CacheableQueryRow createRow(ResultSet cachedRowSet, Set fi int x = combinedString.lastIndexOf(':'); if (x >= 0) { String markingStr = combinedString.substring(0, x); - columnMarkingsMap.put(columnName, AccessExpressionUtil.toAccessExpressionMarkings(ACCESS.newExpression(markingStr))); + columnMarkingsMap.put(columnName, markingFunctions.translateFromColumnVisibility(new ColumnVisibility(markingStr))); columnVisibilityMap.put(columnName, combinedString.substring(x + 1)); } else { - columnMarkingsMap.put(columnName, AccessExpressionUtil.toAccessExpressionMarkings(ACCESS.newExpression(combinedString))); + columnMarkingsMap.put(columnName, markingFunctions.translateFromColumnVisibility(new ColumnVisibility(combinedString))); columnVisibilityMap.put(columnName, ""); } } @@ -123,6 +121,8 @@ public static CacheableQueryRow createRow(ResultSet cachedRowSet, Set fi } catch (SQLException e) { log.error(e.getMessage(), e); + } catch (MarkingFunctions.Exception e) { + throw new RuntimeException(e); } return cqfc; diff --git a/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java b/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java index b18930ca0ff..2c448cc6959 100644 --- a/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java +++ b/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java @@ -1,7 +1,5 @@ package datawave.modification; -import static datawave.marking.AccessExpressionMarkings.ACCESS; - import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -325,7 +323,11 @@ private Markings convertFieldMarkings(Map rawMarkings) { if (vis.isEmpty()) { return null; } - return AccessExpressionUtil.toAccessExpressionMarkings(ACCESS.newExpression(vis)); + try { + return markingFunctions.translateFromColumnVisibility(new ColumnVisibility(vis)); + } catch (MarkingFunctions.Exception e) { + throw new RuntimeException(e); + } } public void process(AccumuloClient client, ModificationRequestBase request, Map> mutableFieldList, Set userAuths, @@ -661,7 +663,7 @@ protected void insert(MultiTableBatchWriter writer, String shardId, String datat if (null == markings || markings.isEmpty()) throw new IllegalArgumentException("No security information specified. Security markings must be supplied"); - viz = AccessExpressionUtil.toColumnVisibility(markings.toAccessExpression()); + viz = markings.toColumnVisibility(); } insert(writer, shardId, datatype, eventUid, viz, fieldName, fieldValue, ts, isIndexOnlyField, isIndexed, isReverseIndexed, dataTypes, false, @@ -872,12 +874,16 @@ protected List> getField(AccumuloClient client, Set markings = AccessExpressionUtil - .toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(e.getKey().getColumnVisibilityParsed())); - if (null != oldFieldMarkings && !oldFieldMarkings.equals(markings)) { - log.trace("Skipping key that does not match with markings: {}", e.getKey()); - continue; + try { + Markings markings = markingFunctions.translateFromColumnVisibility(e.getKey().getColumnVisibilityParsed()); + if (null != oldFieldMarkings && !oldFieldMarkings.equals(markings)) { + log.trace("Skipping key that does not match with markings: {}", e.getKey()); + continue; + } + } catch (MarkingFunctions.Exception e2) { + throw new RuntimeException(e2); } + } results.add(Pair.of(e.getKey(), e.getValue())); } diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java index d0856f0f596..c9789cceb61 100644 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java @@ -1,8 +1,11 @@ package datawave.marking; +import java.nio.charset.StandardCharsets; + import org.apache.accumulo.access.Access; import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.security.ColumnVisibility; +import org.apache.hadoop.io.Text; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonCreator; @@ -47,7 +50,35 @@ public String getColumnVisibilityString() { } @JsonCreator - public static AccessExpressionMarkings create(@JsonProperty(COLUMN_VISIBILITY_KEY) String cv) { - return AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression(cv != null ? cv : "")).build(); + public static AccessExpressionMarkings create(@JsonProperty(COLUMN_VISIBILITY_KEY) String ae) { + return AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression(ae != null ? ae : "")).build(); + } + + public static AccessExpressionMarkings createMarkings(ColumnVisibility cv) { + if (null == cv) { + return null; + } + return create(new String(cv.getExpression(), StandardCharsets.UTF_8)); + } + + public static AccessExpressionMarkings createMarkings(AccessExpression ae) { + if (null == ae) { + return null; + } + return create(ae.getExpression()); + } + + public static AccessExpressionMarkings createMarkings(Text cv) { + if (null == cv) { + return null; + } + return create(cv.toString()); + } + + public static AccessExpressionMarkings createMarkings(String cv) { + if (null == cv) { + return null; + } + return create(cv); } } diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionUtil.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionUtil.java index 850e207b588..0c1a73c440e 100644 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionUtil.java +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionUtil.java @@ -91,17 +91,6 @@ public static AccessExpression toAccessExpression(byte[] visibilityBytes) { return ACCESS.newExpression(new String(visibilityBytes, UTF_8)); } - /** - * Convert a {@link AccessExpression} to an {@link AccessExpressionMarkings}. - * - * @param expression - * The AccessExpression to embed in the wrapper class - * @return the wrapped access expression - */ - public static AccessExpressionMarkings toAccessExpressionMarkings(AccessExpression expression) { - return AccessExpressionMarkings.builder().accessExpression(expression).build(); - } - /** * Normalize an {@link AccessExpression} by deduplicating and sorting terms. This is the replacement for the old {@code ColumnVisibility.flatten()} * behavior. diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java index 556e009e96b..ff00543c5e6 100644 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java @@ -71,14 +71,14 @@ class Default implements MarkingFunctions { @Override public ColumnVisibility combineVisibilities(Collection visibilities) { AccessExpressionMarkings accessExpressionMarkings = combine( - visibilities.stream().map(vis -> AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(vis))) - .collect(Collectors.toList())); + visibilities.stream().map(AccessExpressionMarkings::createMarkings).collect(Collectors.toList())); if (null == accessExpressionMarkings) { + // TODO is this correct behavior, does it match other implementations return new ColumnVisibility(); } - return AccessExpressionUtil.toColumnVisibility(accessExpressionMarkings.toAccessExpression()); + return accessExpressionMarkings.toColumnVisibility(); } @Override diff --git a/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/admin/AdminService.java b/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/admin/AdminService.java index 662f638d00f..f30c189d6a9 100644 --- a/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/admin/AdminService.java +++ b/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/admin/AdminService.java @@ -2,8 +2,6 @@ import static java.nio.charset.StandardCharsets.UTF_8; -import static datawave.marking.AccessExpressionMarkings.ACCESS; - import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -43,7 +41,6 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; -import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.webservice.query.util.OptionallyEncodedString; @@ -573,7 +570,7 @@ public ValidateVisibilityResponse validateVisibilities(String[] visibilityArray) vis.setValid(false); visibilityList.add(vis); try { - Markings markings = AccessExpressionUtil.toAccessExpressionMarkings(ACCESS.newExpression(v)); + Markings markings = markingFunctions.translateFromColumnVisibility(new ColumnVisibility(v)); vis.setVisibility(v); vis.setValid(true); vis.setMarkings(markings); diff --git a/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/lookup/LookupService.java b/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/lookup/LookupService.java index f1c5faffcd2..0a8e49af83e 100644 --- a/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/lookup/LookupService.java +++ b/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/lookup/LookupService.java @@ -2,8 +2,6 @@ import static java.nio.charset.StandardCharsets.UTF_8; -import static datawave.marking.AccessExpressionMarkings.ACCESS; - import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; @@ -20,6 +18,7 @@ import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.commons.codec.binary.Base64; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.builder.ToStringBuilder; @@ -36,7 +35,6 @@ import com.google.common.base.Preconditions; import datawave.accumulo.util.security.UserAuthFunctions; -import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.marking.SecurityMarking; @@ -221,7 +219,7 @@ public LookupResponse lookup(LookupRequest request, DatawaveUserDetails currentU continue; } - final Markings markings = AccessExpressionUtil.toAccessExpressionMarkings(ACCESS.newExpression(k.getColumnVisibility().toString())); + final Markings markings = markingFunctions.translateFromColumnVisibility(new ColumnVisibility(k.getColumnVisibility())); final KeyBase responseKey = responseObjectFactory.createKey(); responseKey.setRow(currRow); diff --git a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java index 9dcf9eeb3c2..64bcae9816f 100644 --- a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java +++ b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java @@ -1,7 +1,5 @@ package datawave.microservice.metadata; -import static datawave.marking.AccessExpressionMarkings.ACCESS; - import java.time.Instant; import java.time.ZoneId; import java.time.format.DateTimeFormatter; @@ -18,6 +16,7 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.hadoop.io.Text; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -25,7 +24,6 @@ import com.google.common.collect.Maps; import datawave.data.ColumnFamilyConstants; -import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.microservice.Connection; import datawave.microservice.dictionary.config.ResponseObjectFactory; @@ -278,7 +276,11 @@ private void setFieldNameAndAlias() { private void setDescriptions() { DefaultDescription description = responseObjectFactory.getDescription(); description.setDescription(currValue.toString()); - description.setMarkings(AccessExpressionUtil.toAccessExpressionMarkings(ACCESS.newExpression(currKey.getColumnVisibility().toString()))); + try { + description.setMarkings(markingFunctions.translateFromColumnVisibility(new ColumnVisibility(currKey.getColumnVisibility()))); + } catch (MarkingFunctions.Exception e) { + throw new RuntimeException(e); + } currField.getDescriptions().add(description); } diff --git a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java index cd793186a47..a48e03faf38 100644 --- a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java +++ b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java @@ -31,7 +31,6 @@ import com.google.common.collect.SetMultimap; import datawave.data.ColumnFamilyConstants; -import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.microservice.dictionary.config.ResponseObjectFactory; @@ -204,6 +203,10 @@ protected SetMultimap loadDescriptions() throws TableNotFoun } private Markings getMarkings(Key k) { - return AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(k.getColumnVisibilityParsed())); + try { + return markingFunctions.translateFromColumnVisibility(k.getColumnVisibilityParsed()); + } catch (MarkingFunctions.Exception e) { + throw new RuntimeException(e); + } } } diff --git a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java index 36c195d232c..d5057e0dc53 100644 --- a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java +++ b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java @@ -62,7 +62,6 @@ import datawave.ingest.mapreduce.job.BulkIngestKey; import datawave.ingest.protobuf.Uid; import datawave.ingest.table.config.TableConfigHelper; -import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.microservice.authorization.user.DatawaveUserDetails; @@ -369,7 +368,7 @@ protected Multimap getEntries(ContentIndexingColumnBasedHan // get markings from metric, otherwise use the default markings Markings markings = updatedQueryMetric.getMarkings(); if (markings != null && !markings.isEmpty()) { - event.setVisibility(AccessExpressionUtil.toColumnVisibility(markings.toAccessExpression())); + event.setVisibility(markings.toColumnVisibility()); } else { event.setVisibility(this.queryMetricHandlerProperties.getDefaultMetricVisibility()); } diff --git a/warehouse/keyword-common/src/main/java/datawave/util/keyword/DefaultTagCloudUtils.java b/warehouse/keyword-common/src/main/java/datawave/util/keyword/DefaultTagCloudUtils.java index 478dac7c2f5..1a039688f7a 100644 --- a/warehouse/keyword-common/src/main/java/datawave/util/keyword/DefaultTagCloudUtils.java +++ b/warehouse/keyword-common/src/main/java/datawave/util/keyword/DefaultTagCloudUtils.java @@ -1,8 +1,7 @@ package datawave.util.keyword; -import static datawave.marking.AccessExpressionMarkings.ACCESS; - import java.io.Serializable; +import java.nio.charset.StandardCharsets; import java.util.Collection; import java.util.Comparator; import java.util.HashMap; @@ -11,11 +10,10 @@ import java.util.Set; import java.util.stream.Collectors; +import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.commons.lang3.StringUtils; -import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; -import datawave.marking.Markings; /** * Default implementations for pluggable utilities for generating tag clouds, includes mechanisms to partition keywords into separate tag clouds, combine or @@ -24,15 +22,24 @@ public class DefaultTagCloudUtils implements TagCloudUtils, Serializable { private static final long serialVersionUID = 652771994052429009L; private static final String MULTI_VALUE_SEPARATOR = ","; + private MarkingFunctions markingFunctions; @Override - public Markings generateCombinedVisibility(Set visibilities) { + public String generateCombinedVisibility(Set visibilities) { if (visibilities == null || visibilities.isEmpty()) { return null; } - Collection> markings = visibilities.stream().map(v -> AccessExpressionUtil.toAccessExpressionMarkings(ACCESS.newExpression(v))) - .collect(Collectors.toSet()); - return (new MarkingFunctions.Default()).combine(markings); + try { + if (null == markingFunctions) { + markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); + } + // Calculate the columnVisibility for this key from the combiner. + ColumnVisibility columnVisibility = markingFunctions + .combineVisibilities(visibilities.stream().map(ColumnVisibility::new).collect(Collectors.toList())); + return new String(columnVisibility.getExpression(), StandardCharsets.UTF_8); + } catch (Exception e) { + throw new RuntimeException(e); + } } @Override diff --git a/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloud.java b/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloud.java index fe563a807e1..aad64d480fa 100644 --- a/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloud.java +++ b/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloud.java @@ -18,9 +18,6 @@ import com.google.gson.Gson; -import datawave.marking.AccessExpressionMarkings; -import datawave.marking.Markings; - /** A tag cloud - a collection of tags that have a keyword, score, frequency and list of sources from which they originated */ public class TagCloud { @@ -41,13 +38,13 @@ public class TagCloud { * @param metadata * metadata for the tag cloud * @param visibility - * the combined access expression for the cloud, or {@code null} + * the combined ColumnVisibility for the cloud, or {@code null} * @param results * the entries that belong in the tag cloud. */ - protected TagCloud(Map metadata, Markings visibility, SortedSet results) { + protected TagCloud(Map metadata, String visibility, SortedSet results) { this.metadata = metadata; - this.visibility = (visibility != null) ? ((AccessExpressionMarkings) visibility).getMarkings().getExpression() : ""; + this.visibility = visibility; this.results = results; } @@ -224,7 +221,7 @@ public List build() { final SortedSet results = new TreeSet<>(comparator); results.addAll(e.getValue()); String partition = getPartition(e.getKey()); - Markings visibility = utils.generateCombinedVisibility(visibilities.get(partition)); + String visibility = utils.generateCombinedVisibility(visibilities.get(partition)); Map tagCloudMetadata = utils.generateCombinedMetadata(this.metadata.get(partition)); tagClouds.add(new TagCloud(tagCloudMetadata, visibility, results)); } diff --git a/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloudUtils.java b/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloudUtils.java index dc837b0ee52..d40d316641d 100644 --- a/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloudUtils.java +++ b/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloudUtils.java @@ -5,8 +5,6 @@ import java.util.Map; import java.util.Set; -import datawave.marking.Markings; - /** * Interface for pluggable utilities for generating tag clouds, includes mechanisms to partition keywords into separate tag clouds, combine or merge visibility * strings, and calculate scores, source collections and frequencies of individual keywords based on observed results @@ -17,9 +15,9 @@ public interface TagCloudUtils { * * @param visibilities * a set of valid visibility strings. - * @return a combined {@link Markings}, or {@code null} if there are no visibilities + * @return a combined visibility string, or {@code null} if there are no visibilities */ - Markings generateCombinedVisibility(Set visibilities); + String generateCombinedVisibility(Set visibilities); /** * Aggregate metadata from multiple sources, combining multi-valued entries into a flattened string diff --git a/warehouse/keyword-common/src/test/resources/MarkingFunctionsContext.xml b/warehouse/keyword-common/src/test/resources/MarkingFunctionsContext.xml new file mode 100644 index 00000000000..a57f6816cab --- /dev/null +++ b/warehouse/keyword-common/src/test/resources/MarkingFunctionsContext.xml @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + diff --git a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryTransformer.java index 74d951bdc17..edba6a92773 100644 --- a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryTransformer.java @@ -12,7 +12,6 @@ import datawave.core.query.cachedresults.CacheableLogic; import datawave.core.query.logic.BaseQueryLogic; import datawave.core.query.logic.BaseQueryLogicTransformer; -import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.microservice.query.Query; @@ -34,6 +33,7 @@ public class DiscoveryTransformer extends BaseQueryLogicTransformer logic, Query settings, QueryModel qm) { + // TODO do any classes inherit from this and require different markingfunctions implemenation super(new MarkingFunctions.Default()); this.markingFunctions = logic.getMarkingFunctions(); this.responseObjectFactory = logic.getResponseObjectFactory(); @@ -46,8 +46,12 @@ public EventBase transform(DiscoveredThing thing) { Preconditions.checkNotNull(thing, "Received a null object to transform!"); EventBase event = this.responseObjectFactory.getEvent(); - Markings markings = AccessExpressionUtil - .toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(new ColumnVisibility(thing.getColumnVisibility()))); + Markings markings; + try { + markings = markingFunctions.translateFromColumnVisibility(new ColumnVisibility(thing.getColumnVisibility())); + } catch (MarkingFunctions.Exception e) { + throw new RuntimeException(e); + } event.setMarkings(markings); List fields = new ArrayList<>(); @@ -63,9 +67,13 @@ public EventBase transform(DiscoveredThing thing) { // If requested return counts separated by colvis, all counts by colvis could be > total record count if (thing.getCountsByColumnVisibility() != null && !thing.getCountsByColumnVisibility().isEmpty()) { for (Map.Entry entry : thing.getCountsByColumnVisibility().entrySet()) { - Markings eMarkings = AccessExpressionUtil - .toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(new ColumnVisibility(entry.getKey().toString()))); - fields.add(this.makeField("RECORD COUNT", eMarkings, entry.getKey().toString(), 0L, entry.getValue().toString())); + Markings recordCountMarkings; + try { + recordCountMarkings = markingFunctions.translateFromColumnVisibility(new ColumnVisibility(entry.getKey().toString())); + } catch (MarkingFunctions.Exception e) { + throw new RuntimeException(e); + } + fields.add(this.makeField("RECORD COUNT", recordCountMarkings, entry.getKey().toString(), 0L, entry.getValue().toString())); } } else { diff --git a/warehouse/query-core/src/main/java/datawave/query/discovery/TermInfoAggregation.java b/warehouse/query-core/src/main/java/datawave/query/discovery/TermInfoAggregation.java index 0adec8c8334..8e760b1fedc 100644 --- a/warehouse/query-core/src/main/java/datawave/query/discovery/TermInfoAggregation.java +++ b/warehouse/query-core/src/main/java/datawave/query/discovery/TermInfoAggregation.java @@ -75,7 +75,7 @@ public DiscoveredThing apply(Collection from) { chosenCount = showReferenceCountInsteadOfTermCount ? referenceCount : termCount; try { - AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(ti.vis)); // just to test + markingFunctions.translateFromColumnVisibility(ti.vis); // just to test // parsing columnVisibilities.add(ti.vis); diff --git a/warehouse/query-core/src/main/java/datawave/query/table/parser/EventKeyValueFactory.java b/warehouse/query-core/src/main/java/datawave/query/table/parser/EventKeyValueFactory.java index 0ada017121a..176bf07beb8 100644 --- a/warehouse/query-core/src/main/java/datawave/query/table/parser/EventKeyValueFactory.java +++ b/warehouse/query-core/src/main/java/datawave/query/table/parser/EventKeyValueFactory.java @@ -1,9 +1,8 @@ package datawave.query.table.parser; import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.security.ColumnVisibility; -import datawave.marking.AccessExpressionUtil; +import datawave.marking.AccessExpressionMarkings; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.query.Constants; @@ -38,8 +37,7 @@ public static EventKeyValue parse(Key key) throws MarkingFunctions.Exception { } protected static void parseColumnVisibility(EventKeyValue event, Key key) { - event.setMarkings(AccessExpressionUtil - .toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(new ColumnVisibility(key.getColumnVisibility())))); + event.setMarkings(AccessExpressionMarkings.createMarkings(key.getColumnVisibility())); } public static class EventKeyValue { diff --git a/warehouse/query-core/src/main/java/datawave/query/table/parser/TermFrequencyKeyValueFactory.java b/warehouse/query-core/src/main/java/datawave/query/table/parser/TermFrequencyKeyValueFactory.java index 615ef7b56e9..a1bfd6894eb 100644 --- a/warehouse/query-core/src/main/java/datawave/query/table/parser/TermFrequencyKeyValueFactory.java +++ b/warehouse/query-core/src/main/java/datawave/query/table/parser/TermFrequencyKeyValueFactory.java @@ -10,7 +10,6 @@ import com.google.protobuf.InvalidProtocolBufferException; import datawave.ingest.protobuf.TermWeight; -import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.query.table.parser.EventKeyValueFactory.EventKeyValue; @@ -57,7 +56,11 @@ public static TermFrequencyKeyValue parse(Key key, Value value, Authorizations a } protected static void parseColumnVisibility(TermFrequencyKeyValue tfkv, Key key, Authorizations auths, MarkingFunctions markingFunctions) { - tfkv.setMarkings(AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(key.getColumnVisibilityParsed()))); + try { + tfkv.setMarkings(markingFunctions.translateFromColumnVisibilityForAuths(key.getColumnVisibilityParsed(), auths)); + } catch (MarkingFunctions.Exception e) { + throw new RuntimeException(e); + } } public static class TermFrequencyKeyValue extends EventKeyValue { diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformer.java index c9bc4d570da..b8e6518e343 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformer.java @@ -16,7 +16,6 @@ import datawave.edge.util.EdgeKey; import datawave.edge.util.EdgeValue; import datawave.edge.util.EdgeValueHelper; -import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.microservice.query.Query; @@ -37,12 +36,11 @@ public EdgeBase transform(Entry entry) { EdgeKey edgeKey = EdgeKey.decode(entry.getKey()); Value value = entry.getValue(); - EdgeBase edge = (EdgeBase) this.responseObjectFactory.getEdge(); + EdgeBase edge = this.responseObjectFactory.getEdge(); boolean statsEdge = edgeKey.isStatsKey(); try { - Markings markings = AccessExpressionUtil - .toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(new ColumnVisibility(edgeKey.getColvis()))); + Markings markings = markingFunctions.translateFromColumnVisibility(new ColumnVisibility(edgeKey.getColvis())); edge.setMarkings(markings); edge.setEdgeType(edgeKey.getType()); edge.setEdgeRelationship(edgeKey.getRelationship()); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java index 587ee33a6b1..cb522739f80 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java @@ -16,8 +16,6 @@ import datawave.core.query.cachedresults.CacheableLogic; import datawave.core.query.logic.BaseQueryLogic; -import datawave.marking.AccessExpressionMarkings; -import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.microservice.query.Query; @@ -31,6 +29,8 @@ public class EventQueryTransformer extends EventQueryTransformerSupport,EventBase> implements CacheableLogic { + private MarkingFunctions markingFunctions; + public EventQueryTransformer(String tableName, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { super(tableName, settings, markingFunctions, responseObjectFactory); } @@ -53,9 +53,7 @@ public EventBase transform(Entry entry) { } EventBase event = this.responseObjectFactory.getEvent(); - Markings markings = AccessExpressionUtil - .toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(new ColumnVisibility(key.getColumnVisibility()))); - if (null == markings || markings.isEmpty()) { + if (null == key.getColumnVisibility() || key.getColumnVisibility().getLength() == 0) { // can't process this one because we did not have valid security markings log.error("Transformer visibility interpreter was null, skipping entry"); return null; @@ -82,9 +80,13 @@ public EventBase transform(Entry entry) { String origFieldName = null; String fieldName = null; - // Hold unique Markings and merge them at the end + // Hold unique visibilities and merge them at the end // for the overall event visibility. - Collection> markingsToMerge = new HashSet<>(); + Collection visibilities = new HashSet<>(); + + if (null == markingFunctions) { + markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); + } for (Entry> e : eventFields.asMap().entrySet()) { origFieldName = e.getKey(); @@ -95,19 +97,36 @@ public EventBase transform(Entry entry) { } for (FieldValue fv : e.getValue()) { - AccessExpressionMarkings fieldMarkings = AccessExpressionUtil - .toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(fv.getVisibility())); - markingsToMerge.add(fieldMarkings); + ColumnVisibility columnVisibility = fv.getVisibility(); + Markings markings; + try { + markings = markingFunctions.translateFromColumnVisibility(columnVisibility); + } catch (MarkingFunctions.Exception ex) { + throw new RuntimeException(ex); + } + visibilities.add(columnVisibility); String value = new String(fv.getValue(), StandardCharsets.UTF_8); // if this is a content field name, then replace the value with the uid if (getContentFieldNames().contains(fieldName)) { value = baseUid; } - values.add(this.makeField(fieldName, fieldMarkings, fieldMarkings.toAccessExpression().getExpression(), entry.getKey().getTimestamp(), value)); + values.add(this.makeField(fieldName, markings, markings.toColumnVisibility(), entry.getKey().getTimestamp(), value)); } } - event.setMarkings((new MarkingFunctions.Default()).combine(markingsToMerge)); + ColumnVisibility cv; + try { + cv = markingFunctions.combineVisibilities(visibilities); + } catch (Exception e) { + log.error("Could not create combined columnVisibility for the count", e); + return null; + } + + try { + event.setMarkings(markingFunctions.translateFromColumnVisibility(cv)); + } catch (MarkingFunctions.Exception e) { + throw new RuntimeException(e); + } event.setFields(new ArrayList<>(values)); Metadata metadata = new Metadata(); @@ -132,7 +151,7 @@ public EventBase transform(Entry entry) { // assign an estimate of the event size // in practice this is about 6 times the size of the kryo bytes - event.setSizeInBytes(entry.getValue().getSize() * 6); + event.setSizeInBytes(entry.getValue().getSize() * 6L); return event; } diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java index 54490282bc3..73754969230 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java @@ -7,6 +7,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import java.util.stream.Collectors; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; @@ -21,7 +22,6 @@ import datawave.data.type.StringType; import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; -import datawave.marking.Markings; import datawave.microservice.query.Query; import datawave.query.attributes.Attribute; import datawave.query.attributes.Attributes; @@ -38,6 +38,8 @@ public class FacetedTransformer extends DocumentTransformerSupport markingFunctions; + /* * By default, assume each cell still has the visibility attached to it */ @@ -128,7 +130,7 @@ protected Collection buildFacets(Key documentKey, String f FieldCardinalityBase fc = this.responseObjectFactory.getFieldCardinality(); fc.setField(v.getFieldName()); - fc.setMarkings(AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(attr.getColumnVisibility()))); + fc.setMarkings(markingFunctions.translateFromColumnVisibility(attr.getColumnVisibility())); fc.setColumnVisibility(AccessExpressionUtil.normalize(fc.getMarkings().toAccessExpression()).getExpression()); fc.setLower(v.getFloorValue()); fc.setUpper(v.getCeilingValue()); @@ -153,7 +155,11 @@ protected FacetsBase buildResponse(Document document, Key documentKey, ColumnVis final Collection documentFields = buildFacets(documentKey, null, document, eventCV, markingFunctions); - facetedResponse.setMarkings(AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(eventCV))); + try { + facetedResponse.setMarkings(markingFunctions.translateFromColumnVisibility(eventCV)); + } catch (MarkingFunctions.Exception e) { + throw new RuntimeException(e); + } facetedResponse.setFields(new ArrayList<>(documentFields)); // assign an estimate of the event size based on the document size @@ -167,7 +173,7 @@ protected FacetsBase buildResponse(Document document, Key documentKey, ColumnVis @Override public BaseQueryResponse createResponse(List resultList) { FacetQueryResponseBase response = responseObjectFactory.getFacetQueryResponse(); - Collection> combinedMarkings = new HashSet<>(); + Collection visibilities = new HashSet<>(); for (Object result : resultList) { FacetsBase facet = (FacetsBase) result; @@ -178,13 +184,28 @@ public BaseQueryResponse createResponse(List resultList) { for (FieldCardinalityBase fcb : facet.getFields()) { if (StringUtils.isNotBlank(fcb.getColumnVisibility())) { - combinedMarkings.add(AccessExpressionUtil - .toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(new ColumnVisibility(fcb.getColumnVisibility())))); + visibilities.add(fcb.getColumnVisibility()); } } } - response.setMarkings((new MarkingFunctions.Default()).combine(combinedMarkings)); + ColumnVisibility cv; + try { + if (null == markingFunctions) { + markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); + } + // Calculate the columnVisibility for this key from the combiner. + cv = markingFunctions.combineVisibilities(visibilities.stream().map(ColumnVisibility::new).collect(Collectors.toList())); + } catch (Exception e) { + log.error("Could not create combined columnVisibility for the count", e); + return null; + } + + try { + response.setMarkings(markingFunctions.translateFromColumnVisibility(cv)); + } catch (MarkingFunctions.Exception e) { + throw new RuntimeException(e); + } return response; } diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/FieldIndexCountQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/FieldIndexCountQueryTransformer.java index b50fcbbcffa..df12f667456 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/FieldIndexCountQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/FieldIndexCountQueryTransformer.java @@ -15,7 +15,6 @@ import datawave.core.query.logic.BaseQueryLogic; import datawave.core.query.logic.BaseQueryLogicTransformer; import datawave.data.hash.UID; -import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.microservice.query.Query; @@ -61,7 +60,12 @@ public EventBase transform(Entry entry) { String key = entry.getKey(); Tuple val = entry.getValue(); - Markings markings = AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(val.getColumnVisibility())); + Markings markings = null; + try { + markings = markingFunctions.translateFromColumnVisibility(val.getColumnVisibility()); + } catch (MarkingFunctions.Exception e) { + throw new RuntimeException(e); + } if (null == markings || markings.isEmpty()) { // can't process this one because we did not have valid security markings log.error("Transformer visibility interpreter was null, skipping entry"); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/GroupingDocumentTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/GroupingDocumentTransformer.java index 22f00a339ce..eb798ad5943 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/GroupingDocumentTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/GroupingDocumentTransformer.java @@ -23,7 +23,6 @@ import datawave.core.query.logic.BaseQueryLogic; import datawave.marking.AccessExpressionMarkings; -import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.microservice.query.Query; @@ -131,9 +130,7 @@ private Multimap getFieldToFieldWithGroupingContextMap(Collection log.trace(this.groupFieldsList + " contains " + shorterName); FieldBase created = null; try { - created = this.makeField(shortName, - AccessExpressionUtil.toAccessExpressionMarkings( - AccessExpressionUtil.toAccessExpression(new ColumnVisibility(field.getColumnVisibility()))), + created = this.makeField(shortName, markingFunctions.translateFromColumnVisibility(new ColumnVisibility(field.getColumnVisibility())), field.getColumnVisibility(), 0L, field.getValueOfTypedValue()); } catch (Exception ex) { log.error(ex); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/ShardIndexQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/ShardIndexQueryTransformer.java index bc0dcecd461..7092a5832d7 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/ShardIndexQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/ShardIndexQueryTransformer.java @@ -18,7 +18,6 @@ import datawave.core.query.logic.BaseQueryLogic; import datawave.core.query.logic.BaseQueryLogicTransformer; import datawave.ingest.protobuf.Uid; -import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.microservice.query.Query; @@ -67,7 +66,12 @@ public EventBase transform(Entry input) { EventBase event = responseObjectFactory.getEvent(); ColumnVisibility columnVisibility = new ColumnVisibility(entry.getKey().getColumnVisibility()); - Markings markings = AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(columnVisibility)); + Markings markings; + try { + markings = markingFunctions.translateFromColumnVisibility(columnVisibility); + } catch (MarkingFunctions.Exception e) { + throw new RuntimeException(e); + } event.setMarkings(markings); List fields = new ArrayList<>(); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/ShardQueryCountTableTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/ShardQueryCountTableTransformer.java index 06e24971ace..f841a947eab 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/ShardQueryCountTableTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/ShardQueryCountTableTransformer.java @@ -47,11 +47,16 @@ public EventBase transform(Entry untypedEntry) { Long count = untypedEntry.getKey(); ColumnVisibility vis = untypedEntry.getValue(); - Markings markings = AccessExpressionUtil.toAccessExpressionMarkings(AccessExpressionUtil.toAccessExpression(vis)); - EventBase e = this.responseObjectFactory.getEvent(); - e.setMarkings(markings); + Markings markings; + try { + markings = markingFunctions.translateFromColumnVisibility(vis); + } catch (MarkingFunctions.Exception ex) { + throw new RuntimeException(ex); + } + + e.setMarkings(markings); FieldBase field = this.makeField(COUNT_CELL, markings, vis, System.currentTimeMillis(), count); e.setMarkings(markings); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/TagCloudTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/TagCloudTransformer.java index 5b72686f8f1..5dadd57ddfa 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/TagCloudTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/TagCloudTransformer.java @@ -10,11 +10,11 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.core.security.ColumnVisibility; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import datawave.core.query.logic.BaseQueryLogicTransformer; -import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.microservice.query.Query; import datawave.query.tables.keyword.KeywordQueryState; @@ -204,7 +204,11 @@ private void configureV2TagCloud(TagCloudBase base, TagCloud tagCloud) { @SuppressWarnings("unchecked") private void convertAndSetMarkings(TagCloudBase base, AccessExpression visibility) { if (visibility != null && !visibility.getExpression().isEmpty()) { - base.setMarkings(AccessExpressionUtil.toAccessExpressionMarkings(visibility)); + try { + base.setMarkings(markingFunctions.translateFromColumnVisibility(new ColumnVisibility(visibility.getExpression()))); + } catch (MarkingFunctions.Exception e) { + throw new RuntimeException(e); + } } } From 172c0d782cf63f88c697f90ae30c96d23de42b52 Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Thu, 7 May 2026 10:25:48 -0400 Subject: [PATCH 35/63] updating comment in MarkingFunctions --- .../src/main/java/datawave/marking/MarkingFunctions.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java index ff00543c5e6..55e6b0771e0 100644 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java @@ -74,7 +74,8 @@ public ColumnVisibility combineVisibilities(Collection visibil visibilities.stream().map(AccessExpressionMarkings::createMarkings).collect(Collectors.toList())); if (null == accessExpressionMarkings) { - // TODO is this correct behavior, does it match other implementations + // is this correct behavior, does it match other implementations? + // switching to null causes lots of NPEs so leaving this as is return new ColumnVisibility(); } From 07d3f4bc7a29308c8cd6d484473fa0f8e4513ba4 Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Thu, 7 May 2026 12:44:28 -0400 Subject: [PATCH 36/63] cleanup --- .../ColumnVisibilitySecurityMarking.java | 2 +- .../cachedresults/CacheableQueryRowImpl.java | 20 ++++++------------- .../modification/MutableMetadataHandler.java | 4 ++-- .../marking/AccessExpressionUtil.java | 6 ++---- .../datawave/marking/MarkingFunctions.java | 8 -------- 5 files changed, 11 insertions(+), 29 deletions(-) diff --git a/core/base-rest-responses/src/main/java/datawave/marking/ColumnVisibilitySecurityMarking.java b/core/base-rest-responses/src/main/java/datawave/marking/ColumnVisibilitySecurityMarking.java index 86512b4cc0d..b4209a676e3 100644 --- a/core/base-rest-responses/src/main/java/datawave/marking/ColumnVisibilitySecurityMarking.java +++ b/core/base-rest-responses/src/main/java/datawave/marking/ColumnVisibilitySecurityMarking.java @@ -97,7 +97,7 @@ public String toAccessExpressionString() { if (null == this.columnVisibility) { return null; } - return toAccessExpression().getExpression(); + return AccessExpressionUtil.normalize(toAccessExpression()).getExpression(); } public void clear() { diff --git a/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java b/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java index 55d420c082e..9db486ef7ff 100644 --- a/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java +++ b/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java @@ -60,23 +60,15 @@ public void addColumn(String columnName, TypedValue columnTypedValue, Markings columnVisibilities = Sets.newHashSet(); - columnVisibilities.add(markingFunctions.translateToColumnVisibility(this.markings)); - columnVisibilities.add(markingFunctions.translateToColumnVisibility(markings)); + columnVisibilities.add(this.markings.toColumnVisibility()); + columnVisibilities.add(markings.toColumnVisibility()); ColumnVisibility combinedVisibility = markingFunctions.combineVisibilities(columnVisibilities); // use combined marking as new markings diff --git a/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java b/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java index 2c448cc6959..89e3dd38acf 100644 --- a/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java +++ b/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java @@ -880,8 +880,8 @@ protected List> getField(AccumuloClient client, Set> { T combine(Markings markings1, Markings markings2) throws MarkingFunctions.Exception; - ColumnVisibility translateToColumnVisibility(Markings markings) throws MarkingFunctions.Exception; - Markings translateFromColumnVisibility(ColumnVisibility columnVisibility) throws MarkingFunctions.Exception; Markings translateFromColumnVisibilityForAuths(ColumnVisibility columnVisibility, Collection authorizations) @@ -138,12 +136,6 @@ public AccessExpressionMarkings combine(Markings markings1, Markings marki return markings1 == null ? null : (AccessExpressionMarkings) markings1; } - @Override - public ColumnVisibility translateToColumnVisibility(Markings markings) { - AccessExpression accessExpression = markings.toAccessExpression(); - return AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.normalize(accessExpression)); - } - @Override public Markings translateFromColumnVisibility(ColumnVisibility expression) { AccessExpression accessExpression = ACCESS.newExpression(new String(expression.getExpression(), StandardCharsets.UTF_8)); From 1358fcdfa438831fd728e10c5e224d8daedd572d Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Thu, 7 May 2026 12:44:45 -0400 Subject: [PATCH 37/63] add unit tests for AccessExpressionMarkings and Utils --- .../marking/AccessExpressionMarkingsTest.java | 179 ++++++++++++++++ .../marking/AccessExpressionUtilTest.java | 194 ++++++++++++++++++ 2 files changed, 373 insertions(+) create mode 100644 core/utils/accumulo-utils/src/test/java/datawave/marking/AccessExpressionMarkingsTest.java create mode 100644 core/utils/accumulo-utils/src/test/java/datawave/marking/AccessExpressionUtilTest.java diff --git a/core/utils/accumulo-utils/src/test/java/datawave/marking/AccessExpressionMarkingsTest.java b/core/utils/accumulo-utils/src/test/java/datawave/marking/AccessExpressionMarkingsTest.java new file mode 100644 index 00000000000..88b412114f0 --- /dev/null +++ b/core/utils/accumulo-utils/src/test/java/datawave/marking/AccessExpressionMarkingsTest.java @@ -0,0 +1,179 @@ +package datawave.marking; + +import static datawave.marking.AccessExpressionMarkings.ACCESS; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.apache.accumulo.access.AccessExpression; +import org.apache.accumulo.core.security.ColumnVisibility; +import org.apache.hadoop.io.Text; +import org.junit.jupiter.api.Test; + +import com.fasterxml.jackson.databind.ObjectMapper; + +public class AccessExpressionMarkingsTest { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + // -- getMarkings / toAccessExpression -- + + @Test + public void testGetMarkingsReturnsAccessExpression() { + AccessExpression ae = ACCESS.newExpression("A&B"); + AccessExpressionMarkings markings = AccessExpressionMarkings.builder().accessExpression(ae).build(); + assertEquals(ae, markings.getMarkings()); + assertEquals(ae, markings.toAccessExpression()); + } + + // -- toColumnVisibility -- + + @Test + public void testToColumnVisibility() { + AccessExpressionMarkings markings = AccessExpressionMarkings.create("B&A"); + ColumnVisibility cv = markings.toColumnVisibility(); + // normalized: sorted + assertEquals("A&B", new String(cv.getExpression())); + } + + // -- isEmpty -- + + @Test + public void testIsEmptyWithNullExpression() { + AccessExpressionMarkings markings = AccessExpressionMarkings.builder().accessExpression(null).build(); + assertTrue(markings.isEmpty()); + } + + @Test + public void testIsEmptyWithEmptyExpression() { + AccessExpressionMarkings markings = AccessExpressionMarkings.create(""); + assertTrue(markings.isEmpty()); + } + + @Test + public void testIsEmptyWithNonEmptyExpression() { + AccessExpressionMarkings markings = AccessExpressionMarkings.create("A"); + assertFalse(markings.isEmpty()); + } + + // -- getColumnVisibilityString -- + + @Test + public void testGetColumnVisibilityString() { + AccessExpressionMarkings markings = AccessExpressionMarkings.create("X|Y"); + assertEquals("X|Y", markings.getColumnVisibilityString()); + } + + @Test + public void testGetColumnVisibilityStringNull() { + AccessExpressionMarkings markings = AccessExpressionMarkings.builder().accessExpression(null).build(); + assertEquals("", markings.getColumnVisibilityString()); + } + + // -- create (JsonCreator) -- + + @Test + public void testCreateWithNull() { + AccessExpressionMarkings markings = AccessExpressionMarkings.create(null); + assertEquals("", markings.getColumnVisibilityString()); + assertTrue(markings.isEmpty()); + } + + @Test + public void testCreateWithExpression() { + AccessExpressionMarkings markings = AccessExpressionMarkings.create("A&B&C"); + assertEquals("A&B&C", markings.getColumnVisibilityString()); + } + + // -- createMarkings(ColumnVisibility) -- + + @Test + public void testCreateMarkingsFromColumnVisibilityNull() { + assertNull(AccessExpressionMarkings.createMarkings((ColumnVisibility) null)); + } + + @Test + public void testCreateMarkingsFromColumnVisibility() { + ColumnVisibility cv = new ColumnVisibility("A|B"); + AccessExpressionMarkings markings = AccessExpressionMarkings.createMarkings(cv); + assertEquals("A|B", markings.getColumnVisibilityString()); + } + + // -- createMarkings(AccessExpression) -- + + @Test + public void testCreateMarkingsFromAccessExpressionNull() { + assertNull(AccessExpressionMarkings.createMarkings((AccessExpression) null)); + } + + @Test + public void testCreateMarkingsFromAccessExpression() { + AccessExpression ae = ACCESS.newExpression("C&D"); + AccessExpressionMarkings markings = AccessExpressionMarkings.createMarkings(ae); + assertEquals("C&D", markings.getColumnVisibilityString()); + } + + // -- createMarkings(Text) -- + + @Test + public void testCreateMarkingsFromTextNull() { + assertNull(AccessExpressionMarkings.createMarkings((Text) null)); + } + + @Test + public void testCreateMarkingsFromText() { + AccessExpressionMarkings markings = AccessExpressionMarkings.createMarkings(new Text("E|F")); + assertEquals("E|F", markings.getColumnVisibilityString()); + } + + // -- createMarkings(String) -- + + @Test + public void testCreateMarkingsFromStringNull() { + assertNull(AccessExpressionMarkings.createMarkings((String) null)); + } + + @Test + public void testCreateMarkingsFromString() { + AccessExpressionMarkings markings = AccessExpressionMarkings.createMarkings("G&H"); + assertEquals("G&H", markings.getColumnVisibilityString()); + } + + // -- JSON round-trip -- + + @Test + public void testJsonSerialization() throws Exception { + AccessExpressionMarkings original = AccessExpressionMarkings.create("A&B"); + String json = MAPPER.writeValueAsString(original); + assertTrue(json.contains("\"columnVisibility\":\"A&B\"")); + } + + @Test + public void testJsonDeserialization() throws Exception { + String json = "{\"@class\":\"datawave.marking.AccessExpressionMarkings\",\"columnVisibility\":\"X|Y\"}"; + AccessExpressionMarkings markings = MAPPER.readValue(json, AccessExpressionMarkings.class); + assertEquals("X|Y", markings.getColumnVisibilityString()); + } + + @Test + public void testJsonRoundTrip() throws Exception { + AccessExpressionMarkings original = AccessExpressionMarkings.create("A&B&C"); + String json = MAPPER.writeValueAsString(original); + // round-trip through the polymorphic Markings type + Markings deserialized = MAPPER.readValue(json, Markings.class); + assertInstanceOf(AccessExpressionMarkings.class, deserialized); + assertEquals(original.getColumnVisibilityString(), ((AccessExpressionMarkings) deserialized).getColumnVisibilityString()); + } + + // -- equals via Lombok @Data -- + + @Test + public void testEquality() { + AccessExpressionMarkings a = AccessExpressionMarkings.create("A&B"); + AccessExpressionMarkings b = AccessExpressionMarkings.create("A&B"); + assertEquals(a, b); + assertEquals(a.hashCode(), b.hashCode()); + } +} diff --git a/core/utils/accumulo-utils/src/test/java/datawave/marking/AccessExpressionUtilTest.java b/core/utils/accumulo-utils/src/test/java/datawave/marking/AccessExpressionUtilTest.java new file mode 100644 index 00000000000..9ff53905d24 --- /dev/null +++ b/core/utils/accumulo-utils/src/test/java/datawave/marking/AccessExpressionUtilTest.java @@ -0,0 +1,194 @@ +package datawave.marking; + +import static datawave.marking.AccessExpressionMarkings.ACCESS; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.apache.accumulo.access.AccessExpression; +import org.apache.accumulo.access.ParsedAccessExpression; +import org.apache.accumulo.core.security.ColumnVisibility; +import org.junit.jupiter.api.Test; + +public class AccessExpressionUtilTest { + + // -- toColumnVisibility -- + + @Test + public void testToColumnVisibilityNull() { + assertNull(AccessExpressionUtil.toColumnVisibility(null)); + } + + @Test + public void testToColumnVisibilityEmpty() { + AccessExpression ae = ACCESS.newExpression(""); + ColumnVisibility cv = AccessExpressionUtil.toColumnVisibility(ae); + assertEquals(0, cv.getExpression().length); + } + + @Test + public void testToColumnVisibilitySimple() { + AccessExpression ae = ACCESS.newExpression("A&B"); + ColumnVisibility cv = AccessExpressionUtil.toColumnVisibility(ae); + assertEquals("A&B", new String(cv.getExpression())); + } + + // -- toAccessExpression(ColumnVisibility) -- + + @Test + public void testToAccessExpressionFromColumnVisibilityNull() { + assertNull(AccessExpressionUtil.toAccessExpression((ColumnVisibility) null)); + } + + @Test + public void testToAccessExpressionFromColumnVisibilityEmpty() { + ColumnVisibility cv = new ColumnVisibility(); + AccessExpression ae = AccessExpressionUtil.toAccessExpression(cv); + assertEquals("", ae.getExpression()); + } + + @Test + public void testToAccessExpressionFromColumnVisibility() { + ColumnVisibility cv = new ColumnVisibility("A|B"); + AccessExpression ae = AccessExpressionUtil.toAccessExpression(cv); + assertEquals("A|B", ae.getExpression()); + } + + // -- toAccessExpression(String) -- + + @Test + public void testToAccessExpressionFromStringNull() { + AccessExpression ae = AccessExpressionUtil.toAccessExpression((String) null); + assertEquals("", ae.getExpression()); + } + + @Test + public void testToAccessExpressionFromStringEmpty() { + AccessExpression ae = AccessExpressionUtil.toAccessExpression(""); + assertEquals("", ae.getExpression()); + } + + @Test + public void testToAccessExpressionFromString() { + AccessExpression ae = AccessExpressionUtil.toAccessExpression("X&Y"); + assertEquals("X&Y", ae.getExpression()); + } + + // -- toAccessExpression(byte[]) -- + + @Test + public void testToAccessExpressionFromBytesNull() { + AccessExpression ae = AccessExpressionUtil.toAccessExpression((byte[]) null); + assertEquals("", ae.getExpression()); + } + + @Test + public void testToAccessExpressionFromBytesEmpty() { + AccessExpression ae = AccessExpressionUtil.toAccessExpression(new byte[0]); + assertEquals("", ae.getExpression()); + } + + @Test + public void testToAccessExpressionFromBytes() { + AccessExpression ae = AccessExpressionUtil.toAccessExpression("C&D".getBytes()); + assertEquals("C&D", ae.getExpression()); + } + + // -- normalize -- + + @Test + public void testNormalizeEmpty() { + AccessExpression ae = ACCESS.newExpression(""); + AccessExpression normalized = AccessExpressionUtil.normalize(ae); + assertEquals("", normalized.getExpression()); + } + + @Test + public void testNormalizeSortsTerms() { + AccessExpression ae = ACCESS.newExpression("Z&Y&X"); + AccessExpression normalized = AccessExpressionUtil.normalize(ae); + assertEquals("X&Y&Z", normalized.getExpression()); + } + + @Test + public void testNormalizeDeduplicates() { + AccessExpression ae = ACCESS.newExpression("A&B&A"); + AccessExpression normalized = AccessExpressionUtil.normalize(ae); + assertEquals("A&B", normalized.getExpression()); + } + + @Test + public void testNormalizeFlattensNestedAnds() { + AccessExpression ae = ACCESS.newExpression("A&(B&C)"); + AccessExpression normalized = AccessExpressionUtil.normalize(ae); + assertEquals("A&B&C", normalized.getExpression()); + } + + @Test + public void testNormalizeSortsOrSubExpressions() { + AccessExpression ae = ACCESS.newExpression("(Z&Y)|(C&B)"); + AccessExpression normalized = AccessExpressionUtil.normalize(ae); + assertEquals("(B&C)|(Y&Z)", normalized.getExpression()); + } + + @Test + public void testNormalizeRemovesUnnecessaryQuotes() { + AccessExpression ae = ACCESS.newExpression("\"ABC\"&\"XYZ\""); + AccessExpression normalized = AccessExpressionUtil.normalize(ae); + assertEquals("ABC&XYZ", normalized.getExpression()); + } + + @Test + public void testNormalizeRemovesUnnecessaryParentheses() { + AccessExpression ae = ACCESS.newExpression("(((ABC)|(XYZ)))"); + AccessExpression normalized = AccessExpressionUtil.normalize(ae); + assertEquals("ABC|XYZ", normalized.getExpression()); + } + + @Test + public void testNormalizeSingleAuth() { + AccessExpression ae = ACCESS.newExpression("ALPHA"); + AccessExpression normalized = AccessExpressionUtil.normalize(ae); + assertEquals("ALPHA", normalized.getExpression()); + } + + // -- emptyExpression / getAccess -- + + @Test + public void testEmptyExpression() { + assertEquals("", AccessExpressionUtil.emptyExpression().getExpression()); + } + + @Test + public void testGetAccess() { + assertEquals(ACCESS, AccessExpressionUtil.getAccess()); + } + + // -- round-trip -- + + @Test + public void testRoundTripColumnVisibilityToAccessExpression() { + ColumnVisibility original = new ColumnVisibility("A&B&C"); + AccessExpression ae = AccessExpressionUtil.toAccessExpression(original); + ColumnVisibility roundTripped = AccessExpressionUtil.toColumnVisibility(ae); + assertEquals(new String(original.getExpression()), new String(roundTripped.getExpression())); + } + + // -- NormalizedExpression -- + + @Test + public void testNormalizedExpressionCompareTo() { + var a = new AccessExpressionUtil.NormalizedExpression("A", ParsedAccessExpression.ExpressionType.AUTHORIZATION); + var b = new AccessExpressionUtil.NormalizedExpression("B", ParsedAccessExpression.ExpressionType.AUTHORIZATION); + assertTrue(a.compareTo(b) < 0); + assertTrue(b.compareTo(a) > 0); + assertEquals(0, a.compareTo(a)); + } + + @Test + public void testNormalizedExpressionEquals() { + var a = new AccessExpressionUtil.NormalizedExpression("A", ParsedAccessExpression.ExpressionType.AUTHORIZATION); + var aDuplicate = new AccessExpressionUtil.NormalizedExpression("A", ParsedAccessExpression.ExpressionType.AUTHORIZATION); + assertEquals(a, aDuplicate); + } +} From eb535f22089132f1e4d43476d539d50978c7a3e6 Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Thu, 7 May 2026 13:22:49 -0400 Subject: [PATCH 38/63] remove instanceof AccessExpressionMarkings usage --- .../datawave/webservice/response/objects/DefaultKey.java | 7 +++---- .../webservice/query/result/event/SimpleField.java | 7 ++++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/microservices/services/accumulo/api/src/main/java/datawave/webservice/response/objects/DefaultKey.java b/microservices/services/accumulo/api/src/main/java/datawave/webservice/response/objects/DefaultKey.java index 83db2a29b8f..cc0698aac78 100644 --- a/microservices/services/accumulo/api/src/main/java/datawave/webservice/response/objects/DefaultKey.java +++ b/microservices/services/accumulo/api/src/main/java/datawave/webservice/response/objects/DefaultKey.java @@ -6,6 +6,7 @@ import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlElement; +import org.apache.accumulo.access.AccessExpression; import org.apache.commons.codec.binary.Base64; import datawave.marking.AccessExpressionMarkings; @@ -87,10 +88,8 @@ public long getTimestamp() { @Override public void setMarkings(Markings markings) { this.markings = markings; - if (markings instanceof AccessExpressionMarkings) { - AccessExpressionMarkings aem = (AccessExpressionMarkings) markings; - this.columnVisibility = new TypedValue(aem.getAccessExpression() != null ? aem.getAccessExpression().getExpression() : ""); - } + AccessExpression ae = markings.toAccessExpression(); + this.columnVisibility = new TypedValue(ae != null ? ae.getExpression() : ""); } @Override diff --git a/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleField.java b/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleField.java index 59f17af0599..111ee9032fc 100644 --- a/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleField.java +++ b/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleField.java @@ -16,6 +16,7 @@ import javax.xml.bind.annotation.XmlElementWrapper; import javax.xml.bind.annotation.XmlTransient; +import org.apache.accumulo.access.AccessExpression; import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; @@ -197,9 +198,9 @@ public void writeTo(Output output, SimpleField message) throws IOException { output.writeObject(4, message.value, message.value.cachedSchema(), false); if (message.markings != null) { // Serialize markings as the expression string for backward compat - if (message.markings instanceof AccessExpressionMarkings) { - String expr = ((AccessExpressionMarkings) message.markings).getAccessExpression().getExpression(); - output.writeString(5, expr, false); + AccessExpression ae = message.markings.toAccessExpression(); + if (ae != null && ae.getExpression() != null) { + output.writeString(5, ae.getExpression(), false); } } } From 8f4fc7221558569815e7d6ce78382e4ee229d3af Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Thu, 7 May 2026 14:15:29 -0400 Subject: [PATCH 39/63] restore translateFromColumnVisibilityForAuths --- .../modification/MutableMetadataHandler.java | 2 +- .../accumulo/admin/AdminService.java | 2 +- .../accumulo/lookup/LookupService.java | 2 +- .../table/parser/ContentKeyValueFactory.java | 6 ++- .../table/parser/EventKeyValueFactory.java | 13 +++-- .../table/parser/KeywordKeyValueFactory.java | 2 +- .../transformer/ContentQueryTransformer.java | 6 ++- .../transformer/EdgeQueryTransformer.java | 2 +- .../transformer/EventQueryTransformer.java | 50 ++++++++++--------- .../query/transformer/FacetedTransformer.java | 2 +- .../FieldIndexCountQueryTransformer.java | 2 +- .../ShardIndexQueryTransformer.java | 4 +- .../ShardQueryCountTableTransformer.java | 6 +-- 13 files changed, 55 insertions(+), 44 deletions(-) diff --git a/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java b/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java index 89e3dd38acf..01527b07bc7 100644 --- a/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java +++ b/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java @@ -875,7 +875,7 @@ protected List> getField(AccumuloClient client, Set markings = markingFunctions.translateFromColumnVisibility(e.getKey().getColumnVisibilityParsed()); + Markings markings = markingFunctions.translateFromColumnVisibilityForAuths(e.getKey().getColumnVisibilityParsed(), userAuths); if (null != oldFieldMarkings && !oldFieldMarkings.equals(markings)) { log.trace("Skipping key that does not match with markings: {}", e.getKey()); continue; diff --git a/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/admin/AdminService.java b/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/admin/AdminService.java index f30c189d6a9..f33bc1707a7 100644 --- a/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/admin/AdminService.java +++ b/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/admin/AdminService.java @@ -570,7 +570,7 @@ public ValidateVisibilityResponse validateVisibilities(String[] visibilityArray) vis.setValid(false); visibilityList.add(vis); try { - Markings markings = markingFunctions.translateFromColumnVisibility(new ColumnVisibility(v)); + Markings markings = markingFunctions.translateFromColumnVisibilityForAuths(new ColumnVisibility(v), authorizations); vis.setVisibility(v); vis.setValid(true); vis.setMarkings(markings); diff --git a/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/lookup/LookupService.java b/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/lookup/LookupService.java index 0a8e49af83e..1677af0f93a 100644 --- a/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/lookup/LookupService.java +++ b/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/lookup/LookupService.java @@ -219,7 +219,7 @@ public LookupResponse lookup(LookupRequest request, DatawaveUserDetails currentU continue; } - final Markings markings = markingFunctions.translateFromColumnVisibility(new ColumnVisibility(k.getColumnVisibility())); + final Markings markings = markingFunctions.translateFromColumnVisibilityForAuths(new ColumnVisibility(k.getColumnVisibility()), mergedAuths); final KeyBase responseKey = responseObjectFactory.createKey(); responseKey.setRow(currRow); diff --git a/warehouse/query-core/src/main/java/datawave/query/table/parser/ContentKeyValueFactory.java b/warehouse/query-core/src/main/java/datawave/query/table/parser/ContentKeyValueFactory.java index 2f09dbc0072..d2de6672545 100644 --- a/warehouse/query-core/src/main/java/datawave/query/table/parser/ContentKeyValueFactory.java +++ b/warehouse/query-core/src/main/java/datawave/query/table/parser/ContentKeyValueFactory.java @@ -7,9 +7,11 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.security.Authorizations; import org.apache.commons.io.IOUtils; import org.apache.log4j.Logger; +import datawave.marking.MarkingFunctions; import datawave.query.Constants; import datawave.query.table.parser.EventKeyValueFactory.EventKeyValue; @@ -17,7 +19,7 @@ public class ContentKeyValueFactory { private static final Logger log = Logger.getLogger(ContentKeyValueFactory.class); - public static ContentKeyValue parse(Key key, Value value) { + public static ContentKeyValue parse(Key key, Value value, Authorizations auths, MarkingFunctions markingFunctions) throws MarkingFunctions.Exception { if (null == key) throw new IllegalArgumentException("Cannot pass null key to ContentKeyValueFactory"); @@ -46,7 +48,7 @@ public static ContentKeyValue parse(Key key, Value value) { c.setContents(contents); } - EventKeyValueFactory.parseColumnVisibility(c, key); + EventKeyValueFactory.parseColumnVisibility(c, key, auths, markingFunctions); return c; } diff --git a/warehouse/query-core/src/main/java/datawave/query/table/parser/EventKeyValueFactory.java b/warehouse/query-core/src/main/java/datawave/query/table/parser/EventKeyValueFactory.java index 176bf07beb8..78a94ce0aa6 100644 --- a/warehouse/query-core/src/main/java/datawave/query/table/parser/EventKeyValueFactory.java +++ b/warehouse/query-core/src/main/java/datawave/query/table/parser/EventKeyValueFactory.java @@ -1,15 +1,17 @@ package datawave.query.table.parser; import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.core.security.ColumnVisibility; -import datawave.marking.AccessExpressionMarkings; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.query.Constants; public class EventKeyValueFactory { - public static EventKeyValue parse(Key key) throws MarkingFunctions.Exception { + public static EventKeyValue parse(Key key, Value value, Authorizations auths, MarkingFunctions markingFunctions) throws MarkingFunctions.Exception { if (null == key) throw new IllegalArgumentException("Cannot pass null key to EventKeyValueFactory"); @@ -31,13 +33,14 @@ public static EventKeyValue parse(Key key) throws MarkingFunctions.Exception { e.setTimestamp(key.getTimestamp()); - parseColumnVisibility(e, key); + parseColumnVisibility(e, key, auths, markingFunctions); return e; } - protected static void parseColumnVisibility(EventKeyValue event, Key key) { - event.setMarkings(AccessExpressionMarkings.createMarkings(key.getColumnVisibility())); + protected static void parseColumnVisibility(EventKeyValue event, Key key, Authorizations auths, MarkingFunctions markingFunctions) + throws MarkingFunctions.Exception { + event.setMarkings(markingFunctions.translateFromColumnVisibilityForAuths(new ColumnVisibility(key.getColumnVisibility()), auths)); } public static class EventKeyValue { diff --git a/warehouse/query-core/src/main/java/datawave/query/table/parser/KeywordKeyValueFactory.java b/warehouse/query-core/src/main/java/datawave/query/table/parser/KeywordKeyValueFactory.java index 332242ae637..0ee5ff66b45 100644 --- a/warehouse/query-core/src/main/java/datawave/query/table/parser/KeywordKeyValueFactory.java +++ b/warehouse/query-core/src/main/java/datawave/query/table/parser/KeywordKeyValueFactory.java @@ -36,7 +36,7 @@ public static KeywordKeyValue parse(Key key, Value value, Authorizations auths, k.setContents(value.get()); } - EventKeyValueFactory.parseColumnVisibility(k, key); + EventKeyValueFactory.parseColumnVisibility(k, key, auths, markingFunctions); return k; } diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/ContentQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/ContentQueryTransformer.java index 3bd93957df9..ac885f55d71 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/ContentQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/ContentQueryTransformer.java @@ -121,7 +121,11 @@ public EventBase transform(Entry entry) { } ContentKeyValue ckv; - ckv = ContentKeyValueFactory.parse(entry.getKey(), entry.getValue()); + try { + ckv = ContentKeyValueFactory.parse(entry.getKey(), entry.getValue(), auths, markingFunctions); + } catch (MarkingFunctions.Exception e1) { + throw new IllegalArgumentException("Unable to parse visibility", e1); + } EventBase e = responseObjectFactory.getEvent(); FieldBase field = responseObjectFactory.getField(); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformer.java index b8e6518e343..8a1d6e36686 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformer.java @@ -40,7 +40,7 @@ public EdgeBase transform(Entry entry) { boolean statsEdge = edgeKey.isStatsKey(); try { - Markings markings = markingFunctions.translateFromColumnVisibility(new ColumnVisibility(edgeKey.getColvis())); + Markings markings = markingFunctions.translateFromColumnVisibilityForAuths(new ColumnVisibility(edgeKey.getColvis()), auths); edge.setMarkings(markings); edge.setEdgeType(edgeKey.getType()); edge.setEdgeRelationship(edgeKey.getRelationship()); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java index cb522739f80..d294e9cbfab 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java @@ -53,7 +53,15 @@ public EventBase transform(Entry entry) { } EventBase event = this.responseObjectFactory.getEvent(); - if (null == key.getColumnVisibility() || key.getColumnVisibility().getLength() == 0) { + Markings markings; + try { + markings = this.markingFunctions.translateFromColumnVisibilityForAuths(new ColumnVisibility(key.getColumnVisibility()), this.auths); + } catch (Exception e) { + log.error("could not translate " + key.getColumnVisibility() + " to markings, skipping entry"); + return null; + } + + if (null == markings || markings.isEmpty()) { // can't process this one because we did not have valid security markings log.error("Transformer visibility interpreter was null, skipping entry"); return null; @@ -82,7 +90,7 @@ public EventBase transform(Entry entry) { // Hold unique visibilities and merge them at the end // for the overall event visibility. - Collection visibilities = new HashSet<>(); + Set visibilitiesToMerge = new HashSet<>(); if (null == markingFunctions) { markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); @@ -97,36 +105,30 @@ public EventBase transform(Entry entry) { } for (FieldValue fv : e.getValue()) { - ColumnVisibility columnVisibility = fv.getVisibility(); - Markings markings; + visibilitiesToMerge.add(fv.getVisibility()); try { - markings = markingFunctions.translateFromColumnVisibility(columnVisibility); - } catch (MarkingFunctions.Exception ex) { - throw new RuntimeException(ex); + Markings fieldMarkings = this.markingFunctions.translateFromColumnVisibility(fv.getVisibility()); + String value = new String(fv.getValue(), StandardCharsets.UTF_8); + // if this is a content field name, then replace the value with the uid + if (getContentFieldNames().contains(fieldName)) { + value = baseUid; + } + values.add(this.makeField(fieldName, fieldMarkings, fieldMarkings.toAccessExpression().getExpression(), entry.getKey().getTimestamp(), + value)); + } catch (MarkingFunctions.Exception e1) { + throw new RuntimeException("could not make markings from: " + fv.getVisibility()); } - visibilities.add(columnVisibility); - String value = new String(fv.getValue(), StandardCharsets.UTF_8); - // if this is a content field name, then replace the value with the uid - if (getContentFieldNames().contains(fieldName)) { - value = baseUid; - } - values.add(this.makeField(fieldName, markings, markings.toColumnVisibility(), entry.getKey().getTimestamp(), value)); } } - ColumnVisibility cv; + ColumnVisibility columnVisibility = null; try { - cv = markingFunctions.combineVisibilities(visibilities); - } catch (Exception e) { - log.error("Could not create combined columnVisibility for the count", e); - return null; + columnVisibility = markingFunctions.combineVisibilities(visibilitiesToMerge); + event.setMarkings(markingFunctions.translateFromColumnVisibility(columnVisibility)); + } catch (Exception e1) { + throw new RuntimeException("could not make markings from: " + columnVisibility); } - try { - event.setMarkings(markingFunctions.translateFromColumnVisibility(cv)); - } catch (MarkingFunctions.Exception e) { - throw new RuntimeException(e); - } event.setFields(new ArrayList<>(values)); Metadata metadata = new Metadata(); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java index 73754969230..f26771cc6c5 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java @@ -130,7 +130,7 @@ protected Collection buildFacets(Key documentKey, String f FieldCardinalityBase fc = this.responseObjectFactory.getFieldCardinality(); fc.setField(v.getFieldName()); - fc.setMarkings(markingFunctions.translateFromColumnVisibility(attr.getColumnVisibility())); + fc.setMarkings(markingFunctions.translateFromColumnVisibilityForAuths(attr.getColumnVisibility(), auths)); fc.setColumnVisibility(AccessExpressionUtil.normalize(fc.getMarkings().toAccessExpression()).getExpression()); fc.setLower(v.getFloorValue()); fc.setUpper(v.getCeilingValue()); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/FieldIndexCountQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/FieldIndexCountQueryTransformer.java index df12f667456..4338887e6cc 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/FieldIndexCountQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/FieldIndexCountQueryTransformer.java @@ -62,7 +62,7 @@ public EventBase transform(Entry entry) { Markings markings = null; try { - markings = markingFunctions.translateFromColumnVisibility(val.getColumnVisibility()); + markings = markingFunctions.translateFromColumnVisibilityForAuths(val.getColumnVisibility(), this.auths); } catch (MarkingFunctions.Exception e) { throw new RuntimeException(e); } diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/ShardIndexQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/ShardIndexQueryTransformer.java index 7092a5832d7..cc0869b6a65 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/ShardIndexQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/ShardIndexQueryTransformer.java @@ -68,9 +68,9 @@ public EventBase transform(Entry input) { ColumnVisibility columnVisibility = new ColumnVisibility(entry.getKey().getColumnVisibility()); Markings markings; try { - markings = markingFunctions.translateFromColumnVisibility(columnVisibility); + markings = markingFunctions.translateFromColumnVisibilityForAuths(columnVisibility, this.auths); } catch (MarkingFunctions.Exception e) { - throw new RuntimeException(e); + throw new RuntimeException("could not make markings from: " + columnVisibility); } event.setMarkings(markings); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/ShardQueryCountTableTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/ShardQueryCountTableTransformer.java index f841a947eab..82c50e5714b 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/ShardQueryCountTableTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/ShardQueryCountTableTransformer.java @@ -51,9 +51,9 @@ public EventBase transform(Entry untypedEntry) { Markings markings; try { - markings = markingFunctions.translateFromColumnVisibility(vis); - } catch (MarkingFunctions.Exception ex) { - throw new RuntimeException(ex); + markings = markingFunctions.translateFromColumnVisibilityForAuths(vis, auths); + } catch (MarkingFunctions.Exception e1) { + throw new IllegalArgumentException("Unable to translate markings", e1); } e.setMarkings(markings); From 2ecb24cad82f2da6a30c2d82f674eca6bc77aff4 Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Thu, 7 May 2026 15:49:03 -0400 Subject: [PATCH 40/63] remove instanceof AccessExpression usage --- .../dictionary/data/DefaultDescription.java | 8 +++----- .../microservice/querymetric/BaseQueryMetric.java | 6 ++---- .../ingest/config/RawRecordContainerImpl.java | 8 +++++--- .../datawave/ingest/mapreduce/SimpleRawRecord.java | 6 +++--- .../webservice/results/cached/CachedResultsBean.java | 12 ++++++++++-- 5 files changed, 23 insertions(+), 17 deletions(-) diff --git a/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java b/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java index 690ae6aa221..73d48f07407 100644 --- a/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java +++ b/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java @@ -55,11 +55,9 @@ public void setMarkings(Markings markings) { if (markings == null || markings.isEmpty()) { this.markingsMap = null; } else { - Object m = markings.getMarkings(); - if (m instanceof AccessExpression) { - this.markingsMap = new HashMap<>(); - this.markingsMap.put(CV_KEY, ((AccessExpression) m).getExpression()); - } + AccessExpression ae = markings.toAccessExpression(); + this.markingsMap = new HashMap<>(); + this.markingsMap.put(CV_KEY, ae.getExpression()); } } diff --git a/microservices/services/query-metric/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java b/microservices/services/query-metric/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java index c6cd45d038b..53a2446bfb8 100644 --- a/microservices/services/query-metric/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java +++ b/microservices/services/query-metric/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java @@ -1173,10 +1173,8 @@ public void setMarkings(Markings markings) { if (markings == null || markings.isEmpty()) { this.columnVisibility = null; } else { - Object m = markings.getMarkings(); - if (m instanceof AccessExpression) { - this.columnVisibility = ((AccessExpression) m).getExpression(); - } + AccessExpression ae = markings.toAccessExpression(); + this.columnVisibility = ae.getExpression(); } } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/config/RawRecordContainerImpl.java b/warehouse/ingest-core/src/main/java/datawave/ingest/config/RawRecordContainerImpl.java index 968d5f4a27e..bd3ddf171d7 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/config/RawRecordContainerImpl.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/config/RawRecordContainerImpl.java @@ -103,9 +103,11 @@ public void setSecurityMarkings(Markings securityMarkings) { protected void syncSecurityMarkingsToFields() { if (securityMarkings != null) { - Object m = securityMarkings.getMarkings(); - if (m instanceof AccessExpression) { - setVisibilityNoSync(((AccessExpression) m).getExpression()); + AccessExpression ae = securityMarkings.toAccessExpression(); + if (ae != null) { + setVisibilityNoSync(ae.getExpression()); + } else { + setVisibilityNoSync(null); } } else { setVisibilityNoSync(null); diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/SimpleRawRecord.java b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/SimpleRawRecord.java index e06de51b000..beae666c961 100644 --- a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/SimpleRawRecord.java +++ b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/SimpleRawRecord.java @@ -240,9 +240,9 @@ public long getDataOutputSize() { public void write(DataOutput dataOutput) throws IOException { String expr = ""; if (securityMarkings != null) { - Object m = securityMarkings.getMarkings(); - if (m instanceof AccessExpression) { - expr = ((AccessExpression) m).getExpression(); + AccessExpression ae = securityMarkings.toAccessExpression(); + if (ae != null) { + expr = ae.getExpression(); } } dataOutput.writeUTF(expr); diff --git a/web-services/cached-results/src/main/java/datawave/webservice/results/cached/CachedResultsBean.java b/web-services/cached-results/src/main/java/datawave/webservice/results/cached/CachedResultsBean.java index 1c36a1e8180..cef2d1a00a6 100644 --- a/web-services/cached-results/src/main/java/datawave/webservice/results/cached/CachedResultsBean.java +++ b/web-services/cached-results/src/main/java/datawave/webservice/results/cached/CachedResultsBean.java @@ -302,8 +302,16 @@ protected void loadBatch(PreparedStatement ps, String owner, String queryId, Str ps.setString(5, cqo.getEventId()); ps.setString(6, cqo.getRow()); ps.setString(7, cqo.getColFam()); - Object rawMarkings = (cqo.getMarkings() != null) ? cqo.getMarkings().getMarkings() : null; - ps.setString(8, (rawMarkings instanceof AccessExpression) ? ((AccessExpression) rawMarkings).getExpression() : ""); + + String visibility = ""; + if (cqo.getMarkings() != null) { + AccessExpression ae = cqo.getMarkings().toAccessExpression(); + if (ae != null) { + visibility = ae.getExpression(); + } + } + + ps.setString(8, visibility); for (Entry e : cqo.getColumnValues().entrySet()) { String columnName = e.getKey(); From 5b757e91b4f1666eac35d67733d0c232c62fa964 Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Thu, 7 May 2026 17:42:37 -0400 Subject: [PATCH 41/63] restore previous exception msgs and handling --- .../iterators/FrequencyMetadataAggregator.java | 2 +- .../query/discovery/DiscoveryTransformer.java | 4 ++-- .../java/datawave/query/jexl/DatawaveInterpreter.java | 6 +++--- .../query/transformer/FacetedTransformer.java | 11 ++++------- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java b/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java index 52469c132e7..d927be96c7f 100644 --- a/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java +++ b/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java @@ -587,7 +587,7 @@ private ColumnVisibility combineAllVisibilities() { try { return markingFunctions.combineVisibilities(visibilities); } catch (MarkingFunctions.Exception e) { - throw new RuntimeException("Could not combine markings", e); + throw new IllegalArgumentException("Failed to combine visibilities " + visibilities, e); } } diff --git a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryTransformer.java index edba6a92773..fbfbbd74e18 100644 --- a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryTransformer.java @@ -50,7 +50,7 @@ public EventBase transform(DiscoveredThing thing) { try { markings = markingFunctions.translateFromColumnVisibility(new ColumnVisibility(thing.getColumnVisibility())); } catch (MarkingFunctions.Exception e) { - throw new RuntimeException(e); + throw new RuntimeException("could not parse to markings: " + thing.getColumnVisibility()); } event.setMarkings(markings); @@ -71,7 +71,7 @@ public EventBase transform(DiscoveredThing thing) { try { recordCountMarkings = markingFunctions.translateFromColumnVisibility(new ColumnVisibility(entry.getKey().toString())); } catch (MarkingFunctions.Exception e) { - throw new RuntimeException(e); + throw new RuntimeException("could not parse to markings: " + thing.getColumnVisibility()); } fields.add(this.makeField("RECORD COUNT", recordCountMarkings, entry.getKey().toString(), 0L, entry.getValue().toString())); diff --git a/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java b/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java index af2a495b866..693b6ea67de 100644 --- a/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java +++ b/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java @@ -475,9 +475,9 @@ private void addHitsForFunction(String field, ASTFunctionNode node, HitListArith try { columnVisibility = markingFunctions.combineVisibilities(columnVisibilities); } catch (MarkingFunctions.Exception e) { - String message = "Could not create combined ColumnVisibility"; - log.error(message, e); - throw new RuntimeException(message, e); + log.error("Failed to combine column visibilities while generating HIT_TERM for phrase function for field [" + field + "]"); + log.error("msg: ", e); + return; } assert columnVisibility != null; diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java index f26771cc6c5..c5ab0b18e30 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java @@ -149,17 +149,13 @@ protected Collection buildFacets(Key documentKey, String f } protected FacetsBase buildResponse(Document document, Key documentKey, ColumnVisibility eventCV, String colf, String row, - MarkingFunctions markingFunctions) { + MarkingFunctions markingFunctions) throws MarkingFunctions.Exception { FacetsBase facetedResponse = responseObjectFactory.getFacets(); final Collection documentFields = buildFacets(documentKey, null, document, eventCV, markingFunctions); - try { - facetedResponse.setMarkings(markingFunctions.translateFromColumnVisibility(eventCV)); - } catch (MarkingFunctions.Exception e) { - throw new RuntimeException(e); - } + facetedResponse.setMarkings(markingFunctions.translateFromColumnVisibility(eventCV)); facetedResponse.setFields(new ArrayList<>(documentFields)); // assign an estimate of the event size based on the document size @@ -204,7 +200,8 @@ public BaseQueryResponse createResponse(List resultList) { try { response.setMarkings(markingFunctions.translateFromColumnVisibility(cv)); } catch (MarkingFunctions.Exception e) { - throw new RuntimeException(e); + log.warn(e); + // original ignored these exceptions } return response; From 0d1bcd33fc48524ed4a971a5158940415e11f46a Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Fri, 8 May 2026 12:15:19 -0400 Subject: [PATCH 42/63] dictionary fixes --- .../dictionary/data/DefaultDescription.java | 10 +++++----- .../webservice/metadata/DefaultMetadataField.java | 13 ++++++------- .../metadata/DefaultMetadataFieldScanner.java | 9 ++------- .../metadata/MetadataDescriptionsHelper.java | 10 +++------- .../metadata/DefaultMetadataFieldScannerTest.java | 2 +- 5 files changed, 17 insertions(+), 27 deletions(-) diff --git a/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java b/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java index 73d48f07407..4f559bacc6c 100644 --- a/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java +++ b/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java @@ -105,7 +105,7 @@ public Schema cachedSchema() { } @XmlTransient - private static final Schema SCHEMA = new Schema() { + private static final Schema SCHEMA = new Schema<>() { public DefaultDescription newMessage() { return new DefaultDescription(); } @@ -130,7 +130,8 @@ public void writeTo(Output output, DefaultDescription message) throws IOExceptio if (message.description != null) { output.writeString(1, message.description, false); } - // markings field 2 no longer written via MapSchema; markingsMap is bridged via Markings + if (message.markingsMap != null) + output.writeObject(1, message.markingsMap, MapSchema.SCHEMA, false); } public void mergeFrom(Input input, DefaultDescription message) throws IOException { @@ -141,9 +142,8 @@ public void mergeFrom(Input input, DefaultDescription message) throws IOExceptio message.description = input.readString(); break; case 2: - // legacy markings map — consume but discard - HashMap legacyMarkings = new HashMap<>(); - input.mergeObject(legacyMarkings, MapSchema.SCHEMA); + message.markingsMap = new HashMap<>(); + input.mergeObject(message.markingsMap, MapSchema.SCHEMA); break; default: input.handleUnknownField(number, this); diff --git a/microservices/services/dictionary/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java b/microservices/services/dictionary/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java index b9221f9b8da..312f07c5284 100644 --- a/microservices/services/dictionary/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java +++ b/microservices/services/dictionary/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java @@ -7,6 +7,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import javax.xml.bind.annotation.XmlAccessType; @@ -72,7 +73,7 @@ public class DefaultMetadataField extends MetadataFieldBase(size); for (int i = 0; i < size; i++) { - // legacy markings map — consume but discard - HashMap legacyMarkings = new HashMap<>(); - input.mergeObject(legacyMarkings, MapSchema.SCHEMA); - message.descriptions.add(new DefaultDescription(input.readString())); + Map markings = new HashMap<>(); + input.mergeObject(markings, MapSchema.SCHEMA); + message.descriptions.add(new DefaultDescription(input.readString(), markings)); } break; case 10: @@ -295,8 +295,7 @@ public void writeTo(Output output, DefaultMetadataField message) throws IOExcept output.writeInt32(9, message.getDescriptions().size(), false); for (DescriptionBase desc : message.getDescriptions()) { output.writeString(9, desc.getDescription(), true); - // Write empty markings map for backward compat with old protostuff readers - output.writeObject(9, new HashMap(), MapSchema.SCHEMA, false); + output.writeObject(9, desc.getMarkingsMap(), MapSchema.SCHEMA, false); } output.writeString(10, message.lastUpdated, false); output.writeBool(11, message.tokenized, false); diff --git a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java index 64bcae9816f..45f62c0b532 100644 --- a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java +++ b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java @@ -16,7 +16,6 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.hadoop.io.Text; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -273,14 +272,10 @@ private void setFieldNameAndAlias() { } // Extract the description from the current value and add it to the current {@link DefaultMetadataField}. - private void setDescriptions() { + private void setDescriptions() throws MarkingFunctions.Exception { DefaultDescription description = responseObjectFactory.getDescription(); description.setDescription(currValue.toString()); - try { - description.setMarkings(markingFunctions.translateFromColumnVisibility(new ColumnVisibility(currKey.getColumnVisibility()))); - } catch (MarkingFunctions.Exception e) { - throw new RuntimeException(e); - } + description.setMarkings(markingFunctions.translateFromColumnVisibility(currKey.getColumnVisibilityParsed())); currField.getDescriptions().add(description); } diff --git a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java index a48e03faf38..c9ce0753a45 100644 --- a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java +++ b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java @@ -177,7 +177,7 @@ public void removeDescription(MetadataEntry entry, DescriptionBase desc) throws } - protected SetMultimap loadDescriptions() throws TableNotFoundException { + protected SetMultimap loadDescriptions() throws TableNotFoundException, MarkingFunctions.Exception { if (log.isTraceEnabled()) log.trace("loadDescriptions from table: " + metadataTableName); // unlike other entries, the desc colf entries have many auths set. We'll use the fullUserAuths in the scanner instead @@ -202,11 +202,7 @@ protected SetMultimap loadDescriptions() throws TableNotFoun } - private Markings getMarkings(Key k) { - try { - return markingFunctions.translateFromColumnVisibility(k.getColumnVisibilityParsed()); - } catch (MarkingFunctions.Exception e) { - throw new RuntimeException(e); - } + private Markings getMarkings(Key k) throws MarkingFunctions.Exception { + return markingFunctions.translateFromColumnVisibility(k.getColumnVisibilityParsed()); } } diff --git a/microservices/services/dictionary/service/src/test/java/datawave/microservice/metadata/DefaultMetadataFieldScannerTest.java b/microservices/services/dictionary/service/src/test/java/datawave/microservice/metadata/DefaultMetadataFieldScannerTest.java index 2e6de88b384..c9af1e44b60 100644 --- a/microservices/services/dictionary/service/src/test/java/datawave/microservice/metadata/DefaultMetadataFieldScannerTest.java +++ b/microservices/services/dictionary/service/src/test/java/datawave/microservice/metadata/DefaultMetadataFieldScannerTest.java @@ -50,7 +50,7 @@ public class DefaultMetadataFieldScannerTest { private static final String[] AUTH = {"PRIVATE"}; private static final Set AUTHS = Collections.singleton(new Authorizations(AUTH)); - private static final ResponseObjectFactory RESPONSE_OBJECT_FACTORY = new ResponseObjectFactory() { + private static final ResponseObjectFactory RESPONSE_OBJECT_FACTORY = new ResponseObjectFactory<>() { @Override public DefaultDataDictionary getDataDictionary() { return null; From 579b0c775e1439392a80afc5f309ae1c7418c7b9 Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Fri, 8 May 2026 14:46:09 -0400 Subject: [PATCH 43/63] mr self-review changes --- core/utils/accumulo-utils/pom.xml | 4 + .../marking/AccessExpressionMarkings.java | 79 ++++++++++++++++++- .../metadata/DefaultMetadataField.java | 2 +- .../handler/atom/AtomDataTypeHandler.java | 14 ++-- .../atom/AtomErrorDataTypeHandler.java | 11 +-- .../handler/error/ErrorDataTypeHandler.java | 11 ++- .../error/ErrorShardedDataTypeHandler.java | 13 +-- .../handler/shard/ShardedDataTypeHandler.java | 8 +- .../data/RawRecordContainerImplTest.java | 2 +- .../ingest/csv/mr/input/CSVRecordReader.java | 30 +++---- .../query/discovery/DiscoveryTransformer.java | 1 - .../jexl/functions/IdentityAggregator.java | 3 +- .../query/predicate/ValueToAttributes.java | 7 +- .../parser/TermFrequencyKeyValueFactory.java | 9 +-- .../shard/CountResultPostprocessor.java | 2 +- .../query/result/event/SimpleEvent.java | 25 +++++- .../query/result/event/SimpleField.java | 3 +- .../query/index/day/TruncatedIndexIngest.java | 3 +- .../datawave/query/util/NoUidIndexIngest.java | 3 +- .../query/result/event/DefaultEvent.java | 18 +++-- .../query/result/event/DefaultField.java | 13 +-- .../result/keyword/DefaultTagCloud.java | 12 +-- 22 files changed, 189 insertions(+), 84 deletions(-) diff --git a/core/utils/accumulo-utils/pom.xml b/core/utils/accumulo-utils/pom.xml index a09c9775a48..1f3693edd06 100644 --- a/core/utils/accumulo-utils/pom.xml +++ b/core/utils/accumulo-utils/pom.xml @@ -62,6 +62,10 @@ + + io.protostuff + protostuff-api + org.apache.accumulo accumulo-access-core diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java index c9789cceb61..5d5f01cd78d 100644 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java @@ -1,6 +1,10 @@ package datawave.marking; +import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.HashMap; + +import javax.xml.bind.annotation.XmlTransient; import org.apache.accumulo.access.Access; import org.apache.accumulo.access.AccessExpression; @@ -11,6 +15,9 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import io.protostuff.Input; +import io.protostuff.Output; +import io.protostuff.Schema; import lombok.Builder; import lombok.Data; @@ -22,7 +29,7 @@ public class AccessExpressionMarkings implements Markings { public static final String COLUMN_VISIBILITY_KEY = "columnVisibility"; - private final AccessExpression accessExpression; + private AccessExpression accessExpression; @Override public AccessExpression getMarkings() { @@ -81,4 +88,74 @@ public static AccessExpressionMarkings createMarkings(String cv) { } return create(cv); } + + @XmlTransient + public static final Schema SCHEMA = new Schema<>() { + private final HashMap fieldMap = new HashMap<>(); + { + fieldMap.put(COLUMN_VISIBILITY_KEY, 1); + } + + @Override + public String getFieldName(int number) { + if (number == 1) { + return COLUMN_VISIBILITY_KEY; + } + return null; + } + + @Override + public int getFieldNumber(String name) { + final Integer number = fieldMap.get(name); + return number == null ? 0 : number; + } + + @Override + public boolean isInitialized(AccessExpressionMarkings message) { + return true; + } + + @Override + public AccessExpressionMarkings newMessage() { + return AccessExpressionMarkings.builder().build(); + } + + @Override + public String messageName() { + return AccessExpressionMarkings.class.getSimpleName(); + } + + @Override + public String messageFullName() { + return AccessExpressionMarkings.class.getName(); + } + + @Override + public Class typeClass() { + return AccessExpressionMarkings.class; + } + + @Override + public void mergeFrom(Input input, AccessExpressionMarkings message) throws IOException { + int number; + while ((number = input.readFieldNumber(this)) != 0) { + if (number == 1) { + String expr = input.readString(); + if (expr != null && !expr.isEmpty()) { + message.accessExpression = ACCESS.newExpression(expr); + } + } else { + input.handleUnknownField(number, this); + } + } + } + + @Override + public void writeTo(Output output, AccessExpressionMarkings message) throws IOException { + if (message.accessExpression != null) { + output.writeString(1, message.accessExpression.getExpression(), false); + } + } + + }; } diff --git a/microservices/services/dictionary/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java b/microservices/services/dictionary/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java index 312f07c5284..e4fd23b0fcd 100644 --- a/microservices/services/dictionary/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java +++ b/microservices/services/dictionary/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java @@ -137,7 +137,7 @@ public Schema cachedSchema() { } @XmlTransient - private static final Schema SCHEMA = new Schema() { + private static final Schema SCHEMA = new Schema<>() { private final HashMap fieldMap = new HashMap<>(); { fieldMap.put("fieldName", 1); diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomDataTypeHandler.java index a91e435ab25..103f61853c9 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomDataTypeHandler.java @@ -29,7 +29,6 @@ import datawave.ingest.mapreduce.job.BulkIngestKey; import datawave.ingest.mapreduce.job.writer.ContextWriter; import datawave.ingest.metadata.RawRecordMetadata; -import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.marking.Markings; import datawave.util.StringUtils; @@ -256,25 +255,24 @@ public Key createKey(String fieldName, String fieldValue, String columnQualifier * the value * @return the visibility */ - - @SuppressWarnings("unchecked") protected String getColumnQualifier(RawRecordContainer event, NormalizedContentInterface value) { + ColumnVisibility visibility = event.getVisibility(); Markings markings = value.getMarkings(); if (markings != null && !markings.isEmpty()) { - return markings.toAccessExpression().getExpression(); + visibility = markings.toColumnVisibility(); } - return flatten(event.getVisibility()); + return flatten(visibility); } /** - * Normalize a ColumnVisibility expression via AccessExpression. + * Create a flattened visibility, using the cache if possible * * @param vis * the column visibility - * @return the normalized visibility string + * @return the flattened visibility */ protected String flatten(ColumnVisibility vis) { - return AccessExpressionUtil.normalize(AccessExpressionUtil.toAccessExpression(vis)).getExpression(); + return new String(markingFunctions.flatten(vis)); } } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomErrorDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomErrorDataTypeHandler.java index 0b490dcdd22..9676e9a8fc2 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomErrorDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomErrorDataTypeHandler.java @@ -54,7 +54,6 @@ public class AtomErrorDataTypeHandler extends AtomDataTyp private AccessExpression defaultVisibility = null; private Configuration conf = null; private ErrorShardedIngestHelper errorHelper = null; - private MarkingFunctions markingFunctions; @Override public void setup(TaskAttemptContext context) { @@ -175,16 +174,18 @@ record = record.copy(); return count; } - @SuppressWarnings("unchecked") protected String getVisibilityColumnString(RawRecordContainer event, NormalizedContentInterface value) { Markings markings = value.getMarkings(); + // value marking is 1st choice if (markings != null && !markings.isEmpty()) { return markings.toAccessExpression().getExpression(); } - ColumnVisibility vis = event.getVisibility(); - if (vis != null) { - return new String(vis.getExpression()); + ColumnVisibility visibility = event.getVisibility(); + if (visibility != null) { + // event marking is 2nd choice + return new String(visibility.getExpression()); } + // default marking is the last resort return defaultVisibility != null ? defaultVisibility.getExpression() : ""; } } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorDataTypeHandler.java index 9ca84e117e1..f50d5592a50 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorDataTypeHandler.java @@ -6,6 +6,7 @@ import java.io.PrintStream; import java.util.Date; +import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.ColumnVisibility; @@ -134,7 +135,12 @@ public void setup(TaskAttemptContext context) { this.conf = context.getConfiguration(); - defaultVisibility = markingsHelper.getDefaultMarkings().toAccessExpression().getExpression().getBytes(UTF_8); + if (markingsHelper != null && markingsHelper.getDefaultMarkings() != null) { + AccessExpression ae = markingsHelper.getDefaultMarkings().toAccessExpression(); + if (ae != null) { + defaultVisibility = ae.getExpression().getBytes(UTF_8); + } + } // Initialize a UID builder based on the configuration uidBuilder = UID.builder(conf); @@ -313,11 +319,10 @@ private Key createKey(String row, Text colf, Text colq, byte[] vis, long ts) { * the value holding the markings * @return the visibility */ - @SuppressWarnings("unchecked") protected byte[] getVisibility(RawRecordContainer event, NormalizedContentInterface value) { byte[] visibility; if (value != null && value.getMarkings() != null && !value.getMarkings().isEmpty()) { - visibility = value.getMarkings().toAccessExpression().getExpression().getBytes(UTF_8); + visibility = flatten(value.getMarkings().toColumnVisibility()); } else if (event.getVisibility() != null) { visibility = flatten(event.getVisibility()); } else { diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java index aa22efdb471..9b613c85c3d 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java @@ -9,6 +9,7 @@ import java.util.Date; import java.util.List; +import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; import org.apache.hadoop.conf.Configurable; @@ -156,10 +157,11 @@ public void setup(TaskAttemptContext context) { tableName = conf.get(ERROR_PROP_PREFIX + SHARD_DINDX_NAME); setShardDictionaryIndexTableName(tableName == null ? null : new Text(tableName)); - try { - defaultVisibility = markingsHelper.getDefaultMarkings().toAccessExpression().getExpression().getBytes(UTF_8); - } catch (Exception e) { - throw new IllegalArgumentException("Failed to parse security marking configuration", e); + if (markingsHelper != null && markingsHelper.getDefaultMarkings() != null) { + AccessExpression ae = markingsHelper.getDefaultMarkings().toAccessExpression(); + if (ae != null) { + defaultVisibility = ae.getExpression().getBytes(UTF_8); + } } log.info("ShardedErrorDataTypeHandler configured."); @@ -343,12 +345,11 @@ public static void getStackTrace(DataOutputBuffer buffer, Throwable e) { * the value * @return the visibility */ - @SuppressWarnings("unchecked") @Override public byte[] getVisibility(RawRecordContainer event, NormalizedContentInterface value) { byte[] visibility; if (value != null && value.getMarkings() != null && !value.getMarkings().isEmpty()) { - visibility = value.getMarkings().toAccessExpression().getExpression().getBytes(UTF_8); + visibility = flatten(value.getMarkings().toColumnVisibility()); } else if (event.getVisibility() != null) { visibility = flatten(event.getVisibility()); } else { diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java index 6d46ed335aa..94ff1a0c06c 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java @@ -1,7 +1,5 @@ package datawave.ingest.mapreduce.handler.shard; -import static java.nio.charset.StandardCharsets.UTF_8; - import java.util.ArrayList; import java.util.Arrays; import java.util.BitSet; @@ -1130,13 +1128,13 @@ public int getOffsetForYearIndex(String shard) { * the entry value * @return the visibility */ - @SuppressWarnings("unchecked") public byte[] getVisibility(RawRecordContainer event, NormalizedContentInterface value) { + ColumnVisibility visibility = event.getVisibility(); Markings markings = value.getMarkings(); if (markings != null && !markings.isEmpty()) { - return markings.toAccessExpression().getExpression().getBytes(UTF_8); + visibility = markings.toColumnVisibility(); } - return flatten(event.getVisibility()); + return flatten(visibility); } /** diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/data/RawRecordContainerImplTest.java b/warehouse/ingest-core/src/test/java/datawave/ingest/data/RawRecordContainerImplTest.java index fb92ff2492a..7201732a8a8 100644 --- a/warehouse/ingest-core/src/test/java/datawave/ingest/data/RawRecordContainerImplTest.java +++ b/warehouse/ingest-core/src/test/java/datawave/ingest/data/RawRecordContainerImplTest.java @@ -398,7 +398,7 @@ public void validate() { if (getVisibility() == null && getSecurityMarkings() == null) { if (markingsHelper != null) { setSecurityMarkings(markingsHelper.getDefaultMarkings()); - } else {} + } } if (getSecurityMarkings() == null) { diff --git a/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java b/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java index 0214745bb89..022c2094200 100644 --- a/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java +++ b/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java @@ -1,7 +1,5 @@ package datawave.ingest.csv.mr.input; -import static datawave.marking.AccessExpressionMarkings.ACCESS; - import java.io.IOException; import java.util.Map; import java.util.Map.Entry; @@ -31,6 +29,8 @@ import datawave.ingest.validation.EventValidator; import datawave.marking.AccessExpressionMarkings; import datawave.marking.AccessExpressionUtil; +import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; public class CSVRecordReader extends CSVReaderBase implements EventFixer { @@ -44,7 +44,7 @@ public class CSVRecordReader extends CSVReaderBase implements EventFixer { private ExtendedCSVHelper csvHelper; private DataTypeOverrideHelper dataTypeHelper; - private String securityMarkingExpression; + private Markings securityMarkings; @Override public void initialize(InputSplit genericSplit, TaskAttemptContext context) throws IOException { @@ -118,7 +118,7 @@ protected ExtendedCSVHelper createHelper(Configuration conf) { @Override public RawRecordContainer getEvent() { - securityMarkingExpression = null; + securityMarkings = null; csvEventId = null; metadataForValidation.clear(); return super.getEvent(); @@ -126,14 +126,9 @@ public RawRecordContainer getEvent() { @Override protected void decorateEvent() { - if (securityMarkingExpression != null) { - try { - AccessExpression ae = AccessExpressionUtil.normalize(ACCESS.newExpression(securityMarkingExpression)); - event.setSecurityMarkings(AccessExpressionMarkings.builder().accessExpression(ae).build()); - } catch (Exception e) { - log.error("Could not set security markings for the event", e); - throw new RuntimeException(e); - } + if (null != this.securityMarkings && !this.securityMarkings.isEmpty()) { + event.setSecurityMarkings(securityMarkings); + event.setVisibility(securityMarkings.toColumnVisibility()); } // now validate @@ -176,14 +171,15 @@ protected void checkField(String fieldName, String fieldValue) { } // If fieldName is a security marking field (as configured by EVENT_SECURITY_MARKING_FIELD_NAMES), - // then put the marking value into this.securityMarkings, where the key is the field name for the marking - // (as configured by EVENT_SECURITY_MARKING_FIELD_DOMAINS) + // then accumulate the marking value into this.securityMarkings if (this.csvHelper.getSecurityMarkingFieldDomainMap().containsKey(fieldName)) { if (!StringUtils.isEmpty(fieldValue)) { - if (securityMarkingExpression == null) { - securityMarkingExpression = fieldValue; + AccessExpression ae = AccessExpressionUtil.normalize(AccessExpressionMarkings.ACCESS.newExpression(fieldValue)); + AccessExpressionMarkings newMarking = AccessExpressionMarkings.builder().accessExpression(ae).build(); + if (securityMarkings == null) { + securityMarkings = newMarking; } else { - securityMarkingExpression = "(" + securityMarkingExpression + ")&(" + fieldValue + ")"; + securityMarkings = new MarkingFunctions.Default().combine(securityMarkings, newMarking); } } } diff --git a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryTransformer.java index fbfbbd74e18..f1ca542b321 100644 --- a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryTransformer.java @@ -33,7 +33,6 @@ public class DiscoveryTransformer extends BaseQueryLogicTransformer logic, Query settings, QueryModel qm) { - // TODO do any classes inherit from this and require different markingfunctions implemenation super(new MarkingFunctions.Default()); this.markingFunctions = logic.getMarkingFunctions(); this.responseObjectFactory = logic.getResponseObjectFactory(); diff --git a/warehouse/query-core/src/main/java/datawave/query/jexl/functions/IdentityAggregator.java b/warehouse/query-core/src/main/java/datawave/query/jexl/functions/IdentityAggregator.java index fee513e7ad7..026189ff52b 100644 --- a/warehouse/query-core/src/main/java/datawave/query/jexl/functions/IdentityAggregator.java +++ b/warehouse/query-core/src/main/java/datawave/query/jexl/functions/IdentityAggregator.java @@ -11,6 +11,7 @@ import org.apache.accumulo.core.iterators.SortedKeyValueIterator; import org.apache.hadoop.io.Text; +import datawave.marking.ColumnVisibilityCache; import datawave.query.attributes.Attribute; import datawave.query.attributes.AttributeFactory; import datawave.query.attributes.Document; @@ -118,7 +119,7 @@ public Key apply(SortedKeyValueIterator itr, Document doc, AttributeF nextKey = (itr.hasTop() ? itr.getTopKey() : null); } - Key docKey = new Key(row, new Text(pointer.toArray()), new Text(), key.getColumnVisibilityParsed(), key.getTimestamp()); + Key docKey = new Key(row, new Text(pointer.toArray()), new Text(), ColumnVisibilityCache.get(key.getColumnVisibilityData()), key.getTimestamp()); Attribute attr = new DocumentKey(docKey, false); doc.put(Document.DOCKEY_FIELD_NAME, attr); diff --git a/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java b/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java index 36cda86b4f5..a5630138e87 100644 --- a/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java +++ b/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java @@ -3,7 +3,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -19,6 +18,7 @@ import com.google.common.collect.Maps; import com.google.common.collect.Multimap; import com.google.common.collect.Multimaps; +import com.google.common.collect.Sets; import datawave.data.type.OneToManyNormalizerType; import datawave.data.type.Type; @@ -90,6 +90,7 @@ public ValueToAttributes(CompositeMetadata compositeMetadata, TypeMetadata typeM public ValueToAttributes(AttributeFactory attributeFactory, CompositeMetadata compositeMetadata, EventDataQueryFilter attrFilter, MarkingFunctions markingFunctions, boolean fromIndex) { this.attrFactory = attributeFactory; + this.markingFunctions = markingFunctions; this.attrFilter = attrFilter; if (compositeMetadata != null) { this.compositeToFieldMap = compositeMetadata.getCompositeFieldMapByType(); @@ -224,7 +225,7 @@ public Attribute getFieldValue(String fieldName, Key k) { } public Attribute joinAttributes(String compositeName, Collection> in, boolean isOverloadedComposite, String separator) throws Exception { - Collection columnVisibilities = new HashSet<>(); + Collection columnVisibilities = Sets.newHashSet(); List dataList = new ArrayList<>(); long timestamp = 0; boolean toKeep = false; @@ -296,7 +297,7 @@ public Attribute joinAttributes(String compositeName, Collection } columnVisibility = markingFunctions.combineVisibilities(columnVisibilities); } catch (java.lang.Exception e) { - log.warn("Invalid columnvisibility after combining!", e); + log.warn("Invalid column visibility after combining!", e); return null; } diff --git a/warehouse/query-core/src/main/java/datawave/query/table/parser/TermFrequencyKeyValueFactory.java b/warehouse/query-core/src/main/java/datawave/query/table/parser/TermFrequencyKeyValueFactory.java index a1bfd6894eb..3582ff20fe5 100644 --- a/warehouse/query-core/src/main/java/datawave/query/table/parser/TermFrequencyKeyValueFactory.java +++ b/warehouse/query-core/src/main/java/datawave/query/table/parser/TermFrequencyKeyValueFactory.java @@ -55,12 +55,9 @@ public static TermFrequencyKeyValue parse(Key key, Value value, Authorizations a return t; } - protected static void parseColumnVisibility(TermFrequencyKeyValue tfkv, Key key, Authorizations auths, MarkingFunctions markingFunctions) { - try { - tfkv.setMarkings(markingFunctions.translateFromColumnVisibilityForAuths(key.getColumnVisibilityParsed(), auths)); - } catch (MarkingFunctions.Exception e) { - throw new RuntimeException(e); - } + protected static void parseColumnVisibility(TermFrequencyKeyValue tfkv, Key key, Authorizations auths, MarkingFunctions markingFunctions) + throws MarkingFunctions.Exception { + tfkv.setMarkings(markingFunctions.translateFromColumnVisibilityForAuths(key.getColumnVisibilityParsed(), auths)); } public static class TermFrequencyKeyValue extends EventKeyValue { diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountResultPostprocessor.java b/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountResultPostprocessor.java index d093279ec55..43c07aaad39 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountResultPostprocessor.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountResultPostprocessor.java @@ -84,7 +84,7 @@ public void apply(List results) { // update the first result FieldBase countField = getCountField(firstResult.getFields()); countField.setValue(count); - countField.setColumnVisibility(new String(columnVisibility.getExpression())); + countField.setColumnVisibility(columnVisibility); } } } diff --git a/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleEvent.java b/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleEvent.java index c87c269e7a0..4dfe7231f12 100644 --- a/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleEvent.java +++ b/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleEvent.java @@ -1,5 +1,7 @@ package datawave.webservice.query.result.event; +import static datawave.marking.AccessExpressionMarkings.ACCESS; + import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; @@ -14,6 +16,10 @@ import javax.xml.bind.annotation.XmlElementWrapper; import javax.xml.bind.annotation.XmlTransient; +import org.apache.accumulo.access.AccessExpression; + +import datawave.marking.AccessExpressionMarkings; +import datawave.marking.Markings; import datawave.webservice.query.data.ObjectSizeOf; import io.protostuff.Input; import io.protostuff.Message; @@ -26,6 +32,9 @@ public class SimpleEvent extends EventBase implements S private static final long serialVersionUID = -1963967131747306439L; + @XmlElementWrapper(name = "Markings") + private Markings markings; + @XmlElement(name = "Metadata") private Metadata metadata = null; @@ -96,7 +105,13 @@ public void writeTo(Output output, SimpleEvent message) throws IOException { output.writeObject(2, field, SimpleField.getSchema(), true); } } - // markings field 3 no longer written; markings are now on the parent EventBase + if (message.markings != null) { + // Serialize markings as the expression string for backward compatibility + AccessExpression ae = message.markings.toAccessExpression(); + if (ae != null && ae.getExpression() != null) { + output.writeString(3, ae.getExpression(), false); + } + } } public void mergeFrom(Input input, SimpleEvent message) throws IOException { @@ -113,9 +128,11 @@ public void mergeFrom(Input input, SimpleEvent message) throws IOException { message.fields.add(f); break; case 3: - // legacy markings map — consume but discard - HashMap legacyMarkings = new HashMap<>(); - input.mergeObject(legacyMarkings, MapSchema.SCHEMA); + String expr = input.readString(); + if (expr != null && !expr.isEmpty()) { + message.markings = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression(expr)).build(); + } + break; default: input.handleUnknownField(number, this); break; diff --git a/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleField.java b/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleField.java index 111ee9032fc..d0971a9ed30 100644 --- a/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleField.java +++ b/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleField.java @@ -197,7 +197,7 @@ public void writeTo(Output output, SimpleField message) throws IOException { if (message.value != null) output.writeObject(4, message.value, message.value.cachedSchema(), false); if (message.markings != null) { - // Serialize markings as the expression string for backward compat + // Serialize markings as the expression string for backward compatibility AccessExpression ae = message.markings.toAccessExpression(); if (ae != null && ae.getExpression() != null) { output.writeString(5, ae.getExpression(), false); @@ -227,6 +227,7 @@ public void mergeFrom(Input input, SimpleField message) throws IOException { if (expr != null && !expr.isEmpty()) { message.markings = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression(expr)).build(); } + break; default: input.handleUnknownField(number, this); break; diff --git a/warehouse/query-core/src/test/java/datawave/query/index/day/TruncatedIndexIngest.java b/warehouse/query-core/src/test/java/datawave/query/index/day/TruncatedIndexIngest.java index 89a2174ed39..3cddd47476d 100644 --- a/warehouse/query-core/src/test/java/datawave/query/index/day/TruncatedIndexIngest.java +++ b/warehouse/query-core/src/test/java/datawave/query/index/day/TruncatedIndexIngest.java @@ -26,6 +26,7 @@ import datawave.ingest.table.aggregator.BitSetCombiner; import datawave.ingest.table.aggregator.util.TruncatedIndexKeyParser; +import datawave.marking.ColumnVisibilityCache; /** * Utility that converts a standard shard index to a truncated index @@ -85,7 +86,7 @@ public void convert(AccumuloClient client, Authorizations auths, String sourceTa for (Map.Entry entry : combined.entries()) { Key k = entry.getKey(); Mutation m = new Mutation(k.getRow()); - ColumnVisibility cv = k.getColumnVisibilityParsed(); + ColumnVisibility cv = ColumnVisibilityCache.get(k.getColumnVisibilityData()); m.put(k.getColumnFamily().toString(), k.getColumnQualifier().toString(), cv, k.getTimestamp(), entry.getValue()); bw.addMutation(m); } diff --git a/warehouse/query-core/src/test/java/datawave/query/util/NoUidIndexIngest.java b/warehouse/query-core/src/test/java/datawave/query/util/NoUidIndexIngest.java index a55e1947006..ff7641b27d0 100644 --- a/warehouse/query-core/src/test/java/datawave/query/util/NoUidIndexIngest.java +++ b/warehouse/query-core/src/test/java/datawave/query/util/NoUidIndexIngest.java @@ -27,6 +27,7 @@ import datawave.ingest.protobuf.Uid; import datawave.ingest.table.aggregator.KeepCountOnlyNoUidAggregator; +import datawave.marking.ColumnVisibilityCache; import datawave.query.index.day.AbstractIndexIngest; /** @@ -79,7 +80,7 @@ public void convert(AccumuloClient client, Authorizations auths, String source, for (Map.Entry entry : combined.entries()) { Key key = entry.getKey(); Mutation m = new Mutation(key.getRow()); - ColumnVisibility cv = key.getColumnVisibilityParsed(); + ColumnVisibility cv = ColumnVisibilityCache.get(key.getColumnVisibilityData()); m.put(key.getColumnFamily(), key.getColumnQualifier(), cv, key.getTimestamp(), entry.getValue()); bw.addMutation(m); } diff --git a/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultEvent.java b/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultEvent.java index c546ca47231..1d1b280850f 100644 --- a/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultEvent.java +++ b/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultEvent.java @@ -16,6 +16,7 @@ import org.apache.commons.lang.builder.EqualsBuilder; +import datawave.marking.AccessExpressionMarkings; import datawave.webservice.query.data.ObjectSizeOf; import io.protostuff.Input; import io.protostuff.Message; @@ -29,11 +30,11 @@ public class DefaultEvent extends EventBase implement private static final long serialVersionUID = 2390592143914560317L; @XmlElement(name = "Metadata") - private Metadata metadata = null; + private Metadata metadata; @XmlElementWrapper(name = "Fields") @XmlElement(name = "Field") - private List fields = null; + private List fields; public List getFields() { return fields; @@ -63,7 +64,7 @@ public boolean equals(Object o) { EqualsBuilder eb = new EqualsBuilder(); - eb.append(this.getMarkings(), v.getMarkings()); + eb.append(this.markings, v.markings); eb.append(this.metadata, v.metadata); eb.append(this.fields, v.fields); return eb.isEquals(); @@ -74,7 +75,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - int result = getMarkings() != null ? getMarkings().hashCode() : 0; + int result = markings != null ? markings.hashCode() : 0; result = 31 * result + (metadata != null ? metadata.hashCode() : 0); result = 31 * result + (fields != null ? fields.hashCode() : 0); return result; @@ -108,6 +109,9 @@ public boolean isInitialized(DefaultEvent message) { } public void writeTo(Output output, DefaultEvent message) throws IOException { + if (message.markings != null) + output.writeObject(1, (AccessExpressionMarkings) message.markings, AccessExpressionMarkings.SCHEMA, false); + if (message.metadata != null) { output.writeObject(2, message.metadata, Metadata.getSchema(), false); } @@ -131,9 +135,7 @@ public void mergeFrom(Input input, DefaultEvent message) throws IOException { while ((number = input.readFieldNumber(this)) != 0) { switch (number) { case 1: - // legacy markings map — consume but discard - HashMap legacyMarkings = new HashMap<>(); - input.mergeObject(legacyMarkings, MapSchema.SCHEMA); + input.mergeObject((AccessExpressionMarkings) message.markings, AccessExpressionMarkings.SCHEMA); break; case 2: message.metadata = input.mergeObject(null, Metadata.getSchema()); @@ -170,7 +172,7 @@ public String getFieldName(int number) { public int getFieldNumber(String name) { final Integer number = fieldMap.get(name); - return number == null ? 0 : number.intValue(); + return number == null ? 0 : number; } final HashMap fieldMap = new HashMap(); diff --git a/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultField.java b/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultField.java index 1d4445b9139..cd8277cc3ff 100644 --- a/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultField.java +++ b/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultField.java @@ -18,6 +18,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import datawave.data.type.Type; +import datawave.marking.AccessExpressionMarkings; import datawave.marking.Markings; import datawave.webservice.query.util.TypedValue; import io.protostuff.Input; @@ -135,7 +136,8 @@ public String getValueString() { @Override public String toString() { StringBuilder buf = new StringBuilder(); - buf.append("Field [columnVisibility=").append(columnVisibility); + buf.append("Field [markings=").append(markings); + buf.append(" columnVisibility=").append(columnVisibility); buf.append(" timestamp=").append(timestamp); buf.append(" name=").append(name); buf.append(" value= "); @@ -148,7 +150,7 @@ public String toString() { @Override public int hashCode() { - return new HashCodeBuilder(17, 37).append(columnVisibility).append(timestamp).append(name).append(value).hashCode(); + return new HashCodeBuilder(17, 37).append(markings).append(columnVisibility).append(timestamp).append(name).append(value).hashCode(); } @Override @@ -160,6 +162,7 @@ public boolean equals(Object o) { eb.append(this.name, v.name); eb.append(this.timestamp, v.timestamp); + eb.append(this.markings, v.markings); eb.append(this.columnVisibility, v.columnVisibility); eb.append(this.value, v.value); return eb.isEquals(); @@ -203,6 +206,8 @@ public boolean isInitialized(DefaultField message) { @Override public void writeTo(Output output, DefaultField message) throws IOException { + if (message.markings != null) + output.writeObject(1, (AccessExpressionMarkings) message.markings, AccessExpressionMarkings.SCHEMA, false); if (message.columnVisibility != null) output.writeString(2, message.columnVisibility, false); output.writeUInt64(3, message.timestamp, false); @@ -218,9 +223,7 @@ public void mergeFrom(Input input, DefaultField message) throws IOException { while ((number = input.readFieldNumber(this)) != 0) { switch (number) { case 1: - // legacy markings map — skip but consume - HashMap legacyMarkings = new HashMap<>(); - input.mergeObject(legacyMarkings, MapSchema.SCHEMA); + input.mergeObject((AccessExpressionMarkings) message.markings, AccessExpressionMarkings.SCHEMA); break; case 2: message.columnVisibility = input.readString(); diff --git a/web-services/client/src/main/java/datawave/webservice/result/keyword/DefaultTagCloud.java b/web-services/client/src/main/java/datawave/webservice/result/keyword/DefaultTagCloud.java index 48457cf59ec..9f8df74deae 100644 --- a/web-services/client/src/main/java/datawave/webservice/result/keyword/DefaultTagCloud.java +++ b/web-services/client/src/main/java/datawave/webservice/result/keyword/DefaultTagCloud.java @@ -18,6 +18,7 @@ import javax.xml.bind.annotation.XmlTransient; import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; +import datawave.marking.AccessExpressionMarkings; import datawave.webservice.query.result.event.MapSchema; import datawave.webservice.xml.util.StringMapAdapter; import io.protostuff.Input; @@ -98,6 +99,9 @@ public boolean isInitialized(DefaultTagCloud message) { } public void writeTo(Output output, DefaultTagCloud message) throws IOException { + if (message.markings != null) + output.writeObject(1, (AccessExpressionMarkings) message.markings, AccessExpressionMarkings.SCHEMA, false); + if (message.language != null) { output.writeString(2, message.language, false); } @@ -125,9 +129,7 @@ public void mergeFrom(Input input, DefaultTagCloud message) throws IOException { while ((number = input.readFieldNumber(this)) != 0) { switch (number) { case 1: - // legacy markings map — consume but discard - HashMap legacyMarkings = new HashMap<>(); - input.mergeObject(legacyMarkings, MapSchema.SCHEMA); + input.mergeObject((AccessExpressionMarkings) message.markings, AccessExpressionMarkings.SCHEMA); break; case 2: message.language = input.readString(); @@ -193,13 +195,13 @@ public boolean equals(Object other) { } DefaultTagCloud otherCloud = (DefaultTagCloud) other; - return Objects.equals(this.getMarkings(), otherCloud.getMarkings()) && Objects.equals(this.language, otherCloud.language) + return Objects.equals(this.markings, otherCloud.markings) && Objects.equals(this.language, otherCloud.language) && Objects.equals(this.metadata, otherCloud.metadata) && this.tags.size() == otherCloud.tags.size() && new HashSet<>(this.tags).containsAll(otherCloud.tags); } @Override public int hashCode() { - return Objects.hash(getMarkings(), language, metadata, tags); + return Objects.hash(markings, language, metadata, tags); } } From 93db2d357cff99a7ff238d24ecdb7d6e764fa8bb Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Mon, 11 May 2026 12:59:01 -0400 Subject: [PATCH 44/63] update microservices to use snapshots --- .../services/accumulo/service/pom.xml | 12 +++++----- microservices/services/audit/service/pom.xml | 10 ++++---- .../services/authorization/service/pom.xml | 10 ++++---- microservices/services/config/pom.xml | 2 +- .../services/dictionary/service/pom.xml | 10 ++++---- .../services/file-provider/service/pom.xml | 2 +- .../services/hazelcast/client/pom.xml | 2 +- .../services/hazelcast/common/pom.xml | 4 ++-- .../services/hazelcast/service/pom.xml | 2 +- .../mapreduce-query/jobs/core/pom.xml | 4 ++-- .../services/mapreduce-query/service/pom.xml | 8 +++---- .../services/modification/service/pom.xml | 12 +++++----- .../services/query-executor/api/pom.xml | 4 ++-- .../services/query-executor/service/pom.xml | 16 ++++++------- .../services/query-metric/api/pom.xml | 4 ++-- .../services/query-metric/service/pom.xml | 20 ++++++++-------- microservices/services/query/service/pom.xml | 22 ++++++++--------- microservices/starters/audit/pom.xml | 8 +++---- microservices/starters/cached-results/pom.xml | 18 +++++++------- microservices/starters/datawave/pom.xml | 8 +++---- microservices/starters/metadata/pom.xml | 6 ++--- microservices/starters/query-metric/pom.xml | 8 +++---- microservices/starters/query/pom.xml | 24 +++++++++---------- 23 files changed, 108 insertions(+), 108 deletions(-) diff --git a/microservices/services/accumulo/service/pom.xml b/microservices/services/accumulo/service/pom.xml index ee4fb5fb79a..03882741831 100644 --- a/microservices/services/accumulo/service/pom.xml +++ b/microservices/services/accumulo/service/pom.xml @@ -24,12 +24,12 @@ 3.20.0 5.2.0 5.2.0 - 7.33.1 - 4.0.1 - 4.0.2 - 4.0.2 - 4.0.7 - 4.0.5 + 7.39.0-SNAPSHOT + 4.0.2-SNAPSHOT + 4.0.3-SNAPSHOT + 4.0.3-SNAPSHOT + 4.0.8-SNAPSHOT + 4.0.6-SNAPSHOT 31.1-jre 3.3.4 4.0.4 diff --git a/microservices/services/audit/service/pom.xml b/microservices/services/audit/service/pom.xml index 529aba9bc89..79f0a33f0e1 100644 --- a/microservices/services/audit/service/pom.xml +++ b/microservices/services/audit/service/pom.xml @@ -22,11 +22,11 @@ 2.18.0 3.20.0 1.9 - 4.0.2 - 4.0.2 - 4.0.4 - 4.0.4 - 4.0.7 + 4.0.3-SNAPSHOT + 4.0.3-SNAPSHOT + 4.0.5-SNAPSHOT + 4.0.5-SNAPSHOT + 4.0.8-SNAPSHOT 2.0.3 31.1-jre 3.3.4 diff --git a/microservices/services/authorization/service/pom.xml b/microservices/services/authorization/service/pom.xml index 695050e9a12..4d722e73425 100644 --- a/microservices/services/authorization/service/pom.xml +++ b/microservices/services/authorization/service/pom.xml @@ -20,11 +20,11 @@ datawave.microservice.authorization.AuthorizationService 3.20.0 - 7.33.1 - 4.0.2 - 4.0.4 - 4.0.4 - 4.0.7 + 7.39.0-SNAPSHOT + 4.0.3-SNAPSHOT + 4.0.5-SNAPSHOT + 4.0.5-SNAPSHOT + 4.0.8-SNAPSHOT 31.1-jre diff --git a/microservices/services/config/pom.xml b/microservices/services/config/pom.xml index 432870d857f..a61fc5cb48b 100644 --- a/microservices/services/config/pom.xml +++ b/microservices/services/config/pom.xml @@ -25,7 +25,7 @@ datawave.microservice.config.server.ConfigServerApplication - 4.0.7 + 4.0.8-SNAPSHOT 5.13.5.202508271544-r 0.2.26 diff --git a/microservices/services/dictionary/service/pom.xml b/microservices/services/dictionary/service/pom.xml index f70bf37f5eb..99dd6f372c8 100644 --- a/microservices/services/dictionary/service/pom.xml +++ b/microservices/services/dictionary/service/pom.xml @@ -28,11 +28,11 @@ 3.2.2 3.20.0 5.2.0 - 7.33.1 - 4.0.2 - 4.0.6 - 4.0.7 - 3.0.4 + 7.39.0-SNAPSHOT + 4.0.3-SNAPSHOT + 4.0.7-SNAPSHOT + 4.0.8-SNAPSHOT + 3.0.5-SNAPSHOT 31.1-jre 3.3.4 4.0.4 diff --git a/microservices/services/file-provider/service/pom.xml b/microservices/services/file-provider/service/pom.xml index 09e562ac03a..932a267d3bb 100644 --- a/microservices/services/file-provider/service/pom.xml +++ b/microservices/services/file-provider/service/pom.xml @@ -19,7 +19,7 @@ datawave.microservice.fileProvider.FileProviderService - 4.0.7 + 4.0.8-SNAPSHOT 2.25.3 diff --git a/microservices/services/hazelcast/client/pom.xml b/microservices/services/hazelcast/client/pom.xml index d758cf38332..872d1baf107 100644 --- a/microservices/services/hazelcast/client/pom.xml +++ b/microservices/services/hazelcast/client/pom.xml @@ -17,7 +17,7 @@ https://github.com/NationalSecurityAgency/datawave - 4.0.4 + 4.0.5-SNAPSHOT 2.2.3 diff --git a/microservices/services/hazelcast/common/pom.xml b/microservices/services/hazelcast/common/pom.xml index 78e092dc680..64e9e1bc3a8 100644 --- a/microservices/services/hazelcast/common/pom.xml +++ b/microservices/services/hazelcast/common/pom.xml @@ -17,8 +17,8 @@ https://github.com/NationalSecurityAgency/datawave - 4.0.7 - 4.0.2 + 4.0.8-SNAPSHOT + 4.0.3-SNAPSHOT 2.25.3 diff --git a/microservices/services/hazelcast/service/pom.xml b/microservices/services/hazelcast/service/pom.xml index 536702fea22..a57672ff84d 100644 --- a/microservices/services/hazelcast/service/pom.xml +++ b/microservices/services/hazelcast/service/pom.xml @@ -19,7 +19,7 @@ datawave.microservice.cached.server.HazelcastService - 4.0.4 + 4.0.5-SNAPSHOT 2.2.3 2.25.3 diff --git a/microservices/services/mapreduce-query/jobs/core/pom.xml b/microservices/services/mapreduce-query/jobs/core/pom.xml index 21fedd6065e..c2d35a97cd3 100644 --- a/microservices/services/mapreduce-query/jobs/core/pom.xml +++ b/microservices/services/mapreduce-query/jobs/core/pom.xml @@ -18,9 +18,9 @@ https://github.com/NationalSecurityAgency/datawave - 7.33.1 + 7.39.0-SNAPSHOT 1.0.0 - 1.0.10 + 1.0.11-SNAPSHOT 3.9.2 diff --git a/microservices/services/mapreduce-query/service/pom.xml b/microservices/services/mapreduce-query/service/pom.xml index d4c0578f458..3a1e927530d 100644 --- a/microservices/services/mapreduce-query/service/pom.xml +++ b/microservices/services/mapreduce-query/service/pom.xml @@ -22,11 +22,11 @@ datawave/ nationalsecurityagency/ datawave.microservice.query.mapreduce.MapReduceQueryService - 7.33.1 - 4.0.2 + 7.39.0-SNAPSHOT + 4.0.3-SNAPSHOT 1.0.5-SNAPSHOT - 4.0.7 - 1.0.10 + 4.0.8-SNAPSHOT + 1.0.11-SNAPSHOT 3.3.4 2.25.3 diff --git a/microservices/services/modification/service/pom.xml b/microservices/services/modification/service/pom.xml index a46db7530b4..6f886eb5c5a 100644 --- a/microservices/services/modification/service/pom.xml +++ b/microservices/services/modification/service/pom.xml @@ -19,12 +19,12 @@ datawave.microservice.modification.ModificationService - 7.33.1 - 4.0.2 - 1.0.1 - 1.0.1 - 4.0.7 - 3.0.4 + 7.39.0-SNAPSHOT + 4.0.3-SNAPSHOT + 1.0.2-SNAPSHOT + 1.0.2-SNAPSHOT + 4.0.8-SNAPSHOT + 3.0.5-SNAPSHOT 4.0.4 2.25.3 diff --git a/microservices/services/query-executor/api/pom.xml b/microservices/services/query-executor/api/pom.xml index 473f0cd7f91..fa4de2e5790 100644 --- a/microservices/services/query-executor/api/pom.xml +++ b/microservices/services/query-executor/api/pom.xml @@ -19,8 +19,8 @@ http://webservice.datawave/v1 3.20.0 - 7.33.1 - 1.0.10 + 7.39.0-SNAPSHOT + 1.0.11-SNAPSHOT 2.3.3 diff --git a/microservices/services/query-executor/service/pom.xml b/microservices/services/query-executor/service/pom.xml index 28bc581f9ea..9b6205d0300 100644 --- a/microservices/services/query-executor/service/pom.xml +++ b/microservices/services/query-executor/service/pom.xml @@ -23,14 +23,14 @@ 4.3 3.20.0 5.2.0 - 7.33.1 - 4.0.2 - 1.0.1 - 1.0.9 - 4.1.3 - 4.0.7 - 1.0.10 - 3.0.5 + 7.39.0-SNAPSHOT + 4.0.3-SNAPSHOT + 1.0.2-SNAPSHOT + 1.0.10-SNAPSHOT + 4.1.4-SNAPSHOT + 4.0.8-SNAPSHOT + 1.0.11-SNAPSHOT + 3.0.6-SNAPSHOT 31.1-jre 3.3.4 4.0.4 diff --git a/microservices/services/query-metric/api/pom.xml b/microservices/services/query-metric/api/pom.xml index 1d4b5acfd39..16d412224da 100644 --- a/microservices/services/query-metric/api/pom.xml +++ b/microservices/services/query-metric/api/pom.xml @@ -21,8 +21,8 @@ ${basedir}/spotbugs-exclude.xml 3.20.0 1.9 - 7.33.1 - 1.0.1 + 7.39.0-SNAPSHOT + 1.0.2-SNAPSHOT 2.3.6 31.1-jre diff --git a/microservices/services/query-metric/service/pom.xml b/microservices/services/query-metric/service/pom.xml index b5a35275860..76502d4ecff 100644 --- a/microservices/services/query-metric/service/pom.xml +++ b/microservices/services/query-metric/service/pom.xml @@ -24,16 +24,16 @@ 1.1.1 3.3 3.20.0 - 7.33.1 - 4.0.1 - 4.0.2 - 4.0.2 - 4.0.4 - 1.0.1 - 4.1.3 - 4.0.7 - 3.0.4 - 3.0.5 + 7.39.0-SNAPSHOT + 4.0.2-SNAPSHOT + 4.0.3-SNAPSHOT + 4.0.3-SNAPSHOT + 4.0.5-SNAPSHOT + 1.0.2-SNAPSHOT + 4.1.4-SNAPSHOT + 4.0.8-SNAPSHOT + 3.0.5-SNAPSHOT + 3.0.6-SNAPSHOT 31.1-jre 3.3.4 2.2.3 diff --git a/microservices/services/query/service/pom.xml b/microservices/services/query/service/pom.xml index 4b37ba0e793..2cbc1cc6a71 100644 --- a/microservices/services/query/service/pom.xml +++ b/microservices/services/query/service/pom.xml @@ -24,17 +24,17 @@ 1.26.0 2.18.0 3.20.0 - 7.33.1 - 4.0.2 - 4.0.2 - 4.0.4 - 1.0.1 - 4.1.3 - 4.0.7 - 4.0.5 - 1.0.12 - 1.0.10 - 3.0.5 + 7.39.0-SNAPSHOT + 4.0.3-SNAPSHOT + 4.0.3-SNAPSHOT + 4.0.5-SNAPSHOT + 1.0.2-SNAPSHOT + 4.1.4-SNAPSHOT + 4.0.8-SNAPSHOT + 4.0.6-SNAPSHOT + 1.0.13-SNAPSHOT + 1.0.11-SNAPSHOT + 3.0.6-SNAPSHOT 31.1-jre 3.3.4 diff --git a/microservices/starters/audit/pom.xml b/microservices/starters/audit/pom.xml index f1b05073a6a..3b3c2b8c1bf 100644 --- a/microservices/starters/audit/pom.xml +++ b/microservices/starters/audit/pom.xml @@ -25,10 +25,10 @@ 1.9.25.1 3.20.0 - 7.33.1 - 4.0.2 - 4.0.2 - 4.0.7 + 7.39.0-SNAPSHOT + 4.0.3-SNAPSHOT + 4.0.3-SNAPSHOT + 4.0.8-SNAPSHOT 31.1-jre 2.2 diff --git a/microservices/starters/cached-results/pom.xml b/microservices/starters/cached-results/pom.xml index 3db9435e9bc..3ca30291339 100644 --- a/microservices/starters/cached-results/pom.xml +++ b/microservices/starters/cached-results/pom.xml @@ -25,15 +25,15 @@ 4.3 3.20.0 - 7.33.1 - 4.0.2 - 4.0.2 - 4.0.4 - 4.0.4 - 1.0.1 - 4.0.7 - 4.0.5 - 1.0.10 + 7.39.0-SNAPSHOT + 4.0.3-SNAPSHOT + 4.0.3-SNAPSHOT + 4.0.5-SNAPSHOT + 4.0.5-SNAPSHOT + 1.0.2-SNAPSHOT + 4.0.8-SNAPSHOT + 4.0.6-SNAPSHOT + 1.0.11-SNAPSHOT 9.3.0 diff --git a/microservices/starters/datawave/pom.xml b/microservices/starters/datawave/pom.xml index b57cd0e2f22..5d5db78abfa 100644 --- a/microservices/starters/datawave/pom.xml +++ b/microservices/starters/datawave/pom.xml @@ -26,10 +26,10 @@ 2.1.4-5792fed3-bulkv2 1.9.25.1 1.9.4 - 7.33.1 - 4.0.2 - 3.0.0 - 4.0.2 + 7.39.0-SNAPSHOT + 4.0.3-SNAPSHOT + 3.0.1-SNAPSHOT + 4.0.3-SNAPSHOT 31.1-jre 2.2 4.0.4 diff --git a/microservices/starters/metadata/pom.xml b/microservices/starters/metadata/pom.xml index 09bd19c7a23..6c589742f65 100644 --- a/microservices/starters/metadata/pom.xml +++ b/microservices/starters/metadata/pom.xml @@ -24,9 +24,9 @@ 2.1.4-5792fed3-bulkv2 - 7.33.1 - 4.0.7 - 4.0.2 + 7.39.0-SNAPSHOT + 4.0.8-SNAPSHOT + 4.0.3-SNAPSHOT 2.2.42 diff --git a/microservices/starters/query-metric/pom.xml b/microservices/starters/query-metric/pom.xml index 71d5a06fd47..5ed87310cce 100644 --- a/microservices/starters/query-metric/pom.xml +++ b/microservices/starters/query-metric/pom.xml @@ -24,10 +24,10 @@ 3.20.0 - 7.33.1 - 4.0.2 - 4.1.3 - 4.0.7 + 7.39.0-SNAPSHOT + 4.0.3-SNAPSHOT + 4.1.4-SNAPSHOT + 4.0.8-SNAPSHOT 31.1-jre diff --git a/microservices/starters/query/pom.xml b/microservices/starters/query/pom.xml index 4185cd7d7e5..608df11e7b8 100644 --- a/microservices/starters/query/pom.xml +++ b/microservices/starters/query/pom.xml @@ -26,18 +26,18 @@ http://webservice.datawave/v1 2.1.4-5792fed3-bulkv2 3.20.0 - 7.33.1 - 4.0.2 - 4.0.2 - 4.0.6 - 4.0.4 - 4.0.4 - 1.0.1 - 1.0.1 - 4.1.3 - 4.0.7 - 3.0.4 - 3.0.5 + 7.39.0-SNAPSHOT + 4.0.3-SNAPSHOT + 4.0.3-SNAPSHOT + 4.0.7-SNAPSHOT + 4.0.5-SNAPSHOT + 4.0.5-SNAPSHOT + 1.0.2-SNAPSHOT + 1.0.2-SNAPSHOT + 4.1.4-SNAPSHOT + 4.0.8-SNAPSHOT + 3.0.5-SNAPSHOT + 3.0.6-SNAPSHOT 31.1-jre 3.3.4 From 8eab1ea1c56496a46f7da216658093ad82f5b82e Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Mon, 11 May 2026 13:09:28 -0400 Subject: [PATCH 45/63] fix ExecutorProperties --- .../query/executor/config/ExecutorProperties.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/microservices/services/query-executor/api/src/main/java/datawave/microservice/query/executor/config/ExecutorProperties.java b/microservices/services/query-executor/api/src/main/java/datawave/microservice/query/executor/config/ExecutorProperties.java index d003a027485..8924975092d 100644 --- a/microservices/services/query-executor/api/src/main/java/datawave/microservice/query/executor/config/ExecutorProperties.java +++ b/microservices/services/query-executor/api/src/main/java/datawave/microservice/query/executor/config/ExecutorProperties.java @@ -2,10 +2,10 @@ import java.util.concurrent.TimeUnit; -import org.springframework.validation.annotation.Validated; +import javax.validation.constraints.NotNull; +import javax.validation.constraints.PositiveOrZero; -import jakarta.validation.constraints.NotNull; -import jakarta.validation.constraints.PositiveOrZero; +import org.springframework.validation.annotation.Validated; @Validated public class ExecutorProperties { From 3e1799115acdef43be2159294f78320931e20a42 Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Mon, 11 May 2026 13:14:51 -0400 Subject: [PATCH 46/63] fix: Disable snapshot usage in POM files --- core/in-memory-accumulo/pom.xml | 2 +- core/metrics-reporter/pom.xml | 2 +- core/utils/accumulo-utils/pom.xml | 2 +- core/utils/common-utils/pom.xml | 2 +- core/utils/metadata-utils/pom.xml | 2 +- microservices/configcheck/pom.xml | 2 +- microservices/microservice-parent/pom.xml | 2 +- microservices/microservice-service-parent/pom.xml | 2 +- microservices/services/accumulo/api/pom.xml | 2 +- microservices/services/accumulo/pom.xml | 2 +- microservices/services/accumulo/service/pom.xml | 2 +- microservices/services/audit/api/pom.xml | 2 +- microservices/services/audit/pom.xml | 2 +- microservices/services/audit/service/pom.xml | 2 +- microservices/services/authorization/api/pom.xml | 2 +- microservices/services/authorization/pom.xml | 2 +- microservices/services/authorization/service/pom.xml | 2 +- microservices/services/config/pom.xml | 2 +- microservices/services/dictionary/api/pom.xml | 2 +- microservices/services/dictionary/pom.xml | 2 +- microservices/services/dictionary/service/pom.xml | 2 +- microservices/services/file-provider/api/pom.xml | 2 +- microservices/services/file-provider/pom.xml | 2 +- microservices/services/file-provider/service/pom.xml | 2 +- microservices/services/hazelcast/client/pom.xml | 2 +- microservices/services/hazelcast/common/pom.xml | 2 +- microservices/services/hazelcast/pom.xml | 2 +- microservices/services/hazelcast/service/pom.xml | 2 +- microservices/services/mapreduce-query/api/pom.xml | 2 +- microservices/services/mapreduce-query/jobs/core/pom.xml | 2 +- microservices/services/mapreduce-query/layout-factory/pom.xml | 2 +- microservices/services/mapreduce-query/pom.xml | 2 +- microservices/services/mapreduce-query/service/pom.xml | 2 +- microservices/services/modification/api/pom.xml | 2 +- microservices/services/modification/pom.xml | 2 +- microservices/services/modification/service/pom.xml | 2 +- microservices/services/query-executor/api/pom.xml | 2 +- microservices/services/query-executor/pom.xml | 2 +- microservices/services/query-executor/service/pom.xml | 2 +- microservices/services/query-metric/api/pom.xml | 2 +- microservices/services/query-metric/pom.xml | 2 +- microservices/services/query-metric/service/pom.xml | 2 +- microservices/services/query/api/pom.xml | 2 +- microservices/services/query/pom.xml | 2 +- microservices/services/query/service/pom.xml | 2 +- microservices/starters/audit/pom.xml | 2 +- microservices/starters/cache/pom.xml | 2 +- microservices/starters/cached-results/pom.xml | 2 +- microservices/starters/datawave/pom.xml | 2 +- microservices/starters/metadata/pom.xml | 2 +- microservices/starters/query-metric/pom.xml | 2 +- microservices/starters/query/pom.xml | 2 +- pom.xml | 2 +- 53 files changed, 53 insertions(+), 53 deletions(-) diff --git a/core/in-memory-accumulo/pom.xml b/core/in-memory-accumulo/pom.xml index 821df23e903..d4372bd8499 100644 --- a/core/in-memory-accumulo/pom.xml +++ b/core/in-memory-accumulo/pom.xml @@ -114,7 +114,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/core/metrics-reporter/pom.xml b/core/metrics-reporter/pom.xml index e5486ef3785..68c2413d033 100644 --- a/core/metrics-reporter/pom.xml +++ b/core/metrics-reporter/pom.xml @@ -105,7 +105,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/core/utils/accumulo-utils/pom.xml b/core/utils/accumulo-utils/pom.xml index 1f3693edd06..9f0dc7571ba 100644 --- a/core/utils/accumulo-utils/pom.xml +++ b/core/utils/accumulo-utils/pom.xml @@ -120,7 +120,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/core/utils/common-utils/pom.xml b/core/utils/common-utils/pom.xml index 4f411581fea..97e5de2c0ea 100644 --- a/core/utils/common-utils/pom.xml +++ b/core/utils/common-utils/pom.xml @@ -41,7 +41,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/core/utils/metadata-utils/pom.xml b/core/utils/metadata-utils/pom.xml index d089c11e3f6..944de13daf9 100644 --- a/core/utils/metadata-utils/pom.xml +++ b/core/utils/metadata-utils/pom.xml @@ -104,7 +104,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/configcheck/pom.xml b/microservices/configcheck/pom.xml index 7ce10cf34ba..1980129e042 100644 --- a/microservices/configcheck/pom.xml +++ b/microservices/configcheck/pom.xml @@ -209,7 +209,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/microservice-parent/pom.xml b/microservices/microservice-parent/pom.xml index 82980cbc5ee..55a8f5ee33c 100644 --- a/microservices/microservice-parent/pom.xml +++ b/microservices/microservice-parent/pom.xml @@ -240,7 +240,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/microservice-service-parent/pom.xml b/microservices/microservice-service-parent/pom.xml index ac28b546adc..c3321d86f10 100644 --- a/microservices/microservice-service-parent/pom.xml +++ b/microservices/microservice-service-parent/pom.xml @@ -136,7 +136,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/accumulo/api/pom.xml b/microservices/services/accumulo/api/pom.xml index 2cc21dbca49..23d1c77ad2b 100644 --- a/microservices/services/accumulo/api/pom.xml +++ b/microservices/services/accumulo/api/pom.xml @@ -126,7 +126,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/accumulo/pom.xml b/microservices/services/accumulo/pom.xml index 1b955053180..77839795115 100644 --- a/microservices/services/accumulo/pom.xml +++ b/microservices/services/accumulo/pom.xml @@ -26,7 +26,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/accumulo/service/pom.xml b/microservices/services/accumulo/service/pom.xml index 03882741831..1d72b90e669 100644 --- a/microservices/services/accumulo/service/pom.xml +++ b/microservices/services/accumulo/service/pom.xml @@ -379,7 +379,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/audit/api/pom.xml b/microservices/services/audit/api/pom.xml index e890fda23d4..fa8058ce07a 100644 --- a/microservices/services/audit/api/pom.xml +++ b/microservices/services/audit/api/pom.xml @@ -71,7 +71,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/audit/pom.xml b/microservices/services/audit/pom.xml index ed3c94b5f55..06f07c9c68b 100644 --- a/microservices/services/audit/pom.xml +++ b/microservices/services/audit/pom.xml @@ -26,7 +26,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/audit/service/pom.xml b/microservices/services/audit/service/pom.xml index 79f0a33f0e1..7e0cfb116a0 100644 --- a/microservices/services/audit/service/pom.xml +++ b/microservices/services/audit/service/pom.xml @@ -441,7 +441,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/authorization/api/pom.xml b/microservices/services/authorization/api/pom.xml index a12c9bba99c..a1769178b2a 100644 --- a/microservices/services/authorization/api/pom.xml +++ b/microservices/services/authorization/api/pom.xml @@ -140,7 +140,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/authorization/pom.xml b/microservices/services/authorization/pom.xml index c4f990f101d..2f5f20b5897 100644 --- a/microservices/services/authorization/pom.xml +++ b/microservices/services/authorization/pom.xml @@ -26,7 +26,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/authorization/service/pom.xml b/microservices/services/authorization/service/pom.xml index 4d722e73425..be1c5a25919 100644 --- a/microservices/services/authorization/service/pom.xml +++ b/microservices/services/authorization/service/pom.xml @@ -256,7 +256,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/config/pom.xml b/microservices/services/config/pom.xml index a61fc5cb48b..2d35b340d7a 100644 --- a/microservices/services/config/pom.xml +++ b/microservices/services/config/pom.xml @@ -138,7 +138,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/dictionary/api/pom.xml b/microservices/services/dictionary/api/pom.xml index 3e9d3723bdd..fa385859928 100644 --- a/microservices/services/dictionary/api/pom.xml +++ b/microservices/services/dictionary/api/pom.xml @@ -106,7 +106,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/dictionary/pom.xml b/microservices/services/dictionary/pom.xml index 729973b3b78..cddd3cd4498 100644 --- a/microservices/services/dictionary/pom.xml +++ b/microservices/services/dictionary/pom.xml @@ -26,7 +26,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/dictionary/service/pom.xml b/microservices/services/dictionary/service/pom.xml index 99dd6f372c8..eb52dee03fc 100644 --- a/microservices/services/dictionary/service/pom.xml +++ b/microservices/services/dictionary/service/pom.xml @@ -371,7 +371,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/file-provider/api/pom.xml b/microservices/services/file-provider/api/pom.xml index e0ce6785c4c..056d4cc263c 100644 --- a/microservices/services/file-provider/api/pom.xml +++ b/microservices/services/file-provider/api/pom.xml @@ -22,7 +22,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/file-provider/pom.xml b/microservices/services/file-provider/pom.xml index eef65e751a3..06499343d28 100644 --- a/microservices/services/file-provider/pom.xml +++ b/microservices/services/file-provider/pom.xml @@ -26,7 +26,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/file-provider/service/pom.xml b/microservices/services/file-provider/service/pom.xml index 932a267d3bb..bdc7325ca40 100644 --- a/microservices/services/file-provider/service/pom.xml +++ b/microservices/services/file-provider/service/pom.xml @@ -66,7 +66,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/hazelcast/client/pom.xml b/microservices/services/hazelcast/client/pom.xml index 872d1baf107..d4dd47e61e7 100644 --- a/microservices/services/hazelcast/client/pom.xml +++ b/microservices/services/hazelcast/client/pom.xml @@ -78,7 +78,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/hazelcast/common/pom.xml b/microservices/services/hazelcast/common/pom.xml index 64e9e1bc3a8..155611205d0 100644 --- a/microservices/services/hazelcast/common/pom.xml +++ b/microservices/services/hazelcast/common/pom.xml @@ -102,7 +102,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/hazelcast/pom.xml b/microservices/services/hazelcast/pom.xml index 2836e95153b..6becacd5acc 100644 --- a/microservices/services/hazelcast/pom.xml +++ b/microservices/services/hazelcast/pom.xml @@ -27,7 +27,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/hazelcast/service/pom.xml b/microservices/services/hazelcast/service/pom.xml index a57672ff84d..e161953f3b2 100644 --- a/microservices/services/hazelcast/service/pom.xml +++ b/microservices/services/hazelcast/service/pom.xml @@ -98,7 +98,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/mapreduce-query/api/pom.xml b/microservices/services/mapreduce-query/api/pom.xml index 5d722fd53a6..97f1c374ad7 100644 --- a/microservices/services/mapreduce-query/api/pom.xml +++ b/microservices/services/mapreduce-query/api/pom.xml @@ -22,7 +22,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/mapreduce-query/jobs/core/pom.xml b/microservices/services/mapreduce-query/jobs/core/pom.xml index c2d35a97cd3..5e2e144846f 100644 --- a/microservices/services/mapreduce-query/jobs/core/pom.xml +++ b/microservices/services/mapreduce-query/jobs/core/pom.xml @@ -106,7 +106,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/mapreduce-query/layout-factory/pom.xml b/microservices/services/mapreduce-query/layout-factory/pom.xml index de3dea9911c..f1123c8fef9 100644 --- a/microservices/services/mapreduce-query/layout-factory/pom.xml +++ b/microservices/services/mapreduce-query/layout-factory/pom.xml @@ -31,7 +31,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/mapreduce-query/pom.xml b/microservices/services/mapreduce-query/pom.xml index b121b57353c..13bf967f40f 100644 --- a/microservices/services/mapreduce-query/pom.xml +++ b/microservices/services/mapreduce-query/pom.xml @@ -26,7 +26,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/mapreduce-query/service/pom.xml b/microservices/services/mapreduce-query/service/pom.xml index 3a1e927530d..6d992d8c81e 100644 --- a/microservices/services/mapreduce-query/service/pom.xml +++ b/microservices/services/mapreduce-query/service/pom.xml @@ -265,7 +265,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/modification/api/pom.xml b/microservices/services/modification/api/pom.xml index 21f560fb6e0..fb2d35acd0a 100644 --- a/microservices/services/modification/api/pom.xml +++ b/microservices/services/modification/api/pom.xml @@ -76,7 +76,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/modification/pom.xml b/microservices/services/modification/pom.xml index 92d0e3f87f4..290e3a65e7d 100644 --- a/microservices/services/modification/pom.xml +++ b/microservices/services/modification/pom.xml @@ -26,7 +26,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/modification/service/pom.xml b/microservices/services/modification/service/pom.xml index 6f886eb5c5a..5ae8e7169ce 100644 --- a/microservices/services/modification/service/pom.xml +++ b/microservices/services/modification/service/pom.xml @@ -250,7 +250,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/query-executor/api/pom.xml b/microservices/services/query-executor/api/pom.xml index fa4de2e5790..f3032f9ded9 100644 --- a/microservices/services/query-executor/api/pom.xml +++ b/microservices/services/query-executor/api/pom.xml @@ -86,7 +86,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/query-executor/pom.xml b/microservices/services/query-executor/pom.xml index 9368cbd73dc..5d030feeb5f 100644 --- a/microservices/services/query-executor/pom.xml +++ b/microservices/services/query-executor/pom.xml @@ -26,7 +26,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/query-executor/service/pom.xml b/microservices/services/query-executor/service/pom.xml index 9b6205d0300..b41a37e8f47 100644 --- a/microservices/services/query-executor/service/pom.xml +++ b/microservices/services/query-executor/service/pom.xml @@ -554,7 +554,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/query-metric/api/pom.xml b/microservices/services/query-metric/api/pom.xml index 16d412224da..19da3c53d5c 100644 --- a/microservices/services/query-metric/api/pom.xml +++ b/microservices/services/query-metric/api/pom.xml @@ -223,7 +223,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/query-metric/pom.xml b/microservices/services/query-metric/pom.xml index e5700e00c6e..765d183d4c3 100644 --- a/microservices/services/query-metric/pom.xml +++ b/microservices/services/query-metric/pom.xml @@ -26,7 +26,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/query-metric/service/pom.xml b/microservices/services/query-metric/service/pom.xml index 76502d4ecff..96b837371f2 100644 --- a/microservices/services/query-metric/service/pom.xml +++ b/microservices/services/query-metric/service/pom.xml @@ -675,7 +675,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/query/api/pom.xml b/microservices/services/query/api/pom.xml index 28cc310bb35..652a277eee0 100644 --- a/microservices/services/query/api/pom.xml +++ b/microservices/services/query/api/pom.xml @@ -138,7 +138,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/query/pom.xml b/microservices/services/query/pom.xml index 8526739bc95..369c2076ced 100644 --- a/microservices/services/query/pom.xml +++ b/microservices/services/query/pom.xml @@ -26,7 +26,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/services/query/service/pom.xml b/microservices/services/query/service/pom.xml index 2cbc1cc6a71..8278519b7ec 100644 --- a/microservices/services/query/service/pom.xml +++ b/microservices/services/query/service/pom.xml @@ -601,7 +601,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/starters/audit/pom.xml b/microservices/starters/audit/pom.xml index 3b3c2b8c1bf..ec8c6b89548 100644 --- a/microservices/starters/audit/pom.xml +++ b/microservices/starters/audit/pom.xml @@ -173,7 +173,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/starters/cache/pom.xml b/microservices/starters/cache/pom.xml index 42c4692d1a5..01ab2bcb467 100644 --- a/microservices/starters/cache/pom.xml +++ b/microservices/starters/cache/pom.xml @@ -94,7 +94,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/starters/cached-results/pom.xml b/microservices/starters/cached-results/pom.xml index 3ca30291339..ac47212c3c4 100644 --- a/microservices/starters/cached-results/pom.xml +++ b/microservices/starters/cached-results/pom.xml @@ -300,7 +300,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/starters/datawave/pom.xml b/microservices/starters/datawave/pom.xml index 5d5db78abfa..03d2c75677a 100644 --- a/microservices/starters/datawave/pom.xml +++ b/microservices/starters/datawave/pom.xml @@ -648,7 +648,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/starters/metadata/pom.xml b/microservices/starters/metadata/pom.xml index 6c589742f65..a621f5fed05 100644 --- a/microservices/starters/metadata/pom.xml +++ b/microservices/starters/metadata/pom.xml @@ -152,7 +152,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/starters/query-metric/pom.xml b/microservices/starters/query-metric/pom.xml index 5ed87310cce..ca61ffe3afa 100644 --- a/microservices/starters/query-metric/pom.xml +++ b/microservices/starters/query-metric/pom.xml @@ -151,7 +151,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/microservices/starters/query/pom.xml b/microservices/starters/query/pom.xml index 608df11e7b8..8037ad78d46 100644 --- a/microservices/starters/query/pom.xml +++ b/microservices/starters/query/pom.xml @@ -520,7 +520,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/pom.xml b/pom.xml index 62af22d512e..ed1064f965d 100644 --- a/pom.xml +++ b/pom.xml @@ -1601,7 +1601,7 @@ true - true + false github-datawave https://maven.pkg.github.com/NationalSecurityAgency/datawave From dbdf6cb66ac227d392eb4fee115e776318b3043e Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Mon, 11 May 2026 13:20:01 -0400 Subject: [PATCH 47/63] revert microservice parents --- core/metrics-reporter/pom.xml | 2 +- microservices/services/file-provider/pom.xml | 2 +- microservices/services/hazelcast/pom.xml | 2 +- microservices/services/mapreduce-query/pom.xml | 2 +- microservices/services/modification/pom.xml | 2 +- microservices/services/query-executor/api/pom.xml | 2 +- microservices/services/query-executor/pom.xml | 2 +- microservices/services/query-executor/service/pom.xml | 2 +- microservices/services/query-metric/pom.xml | 2 +- microservices/services/query/pom.xml | 2 +- microservices/services/query/service/pom.xml | 2 +- microservices/starters/cached-results/pom.xml | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/core/metrics-reporter/pom.xml b/core/metrics-reporter/pom.xml index 68c2413d033..475fe34bd5c 100644 --- a/core/metrics-reporter/pom.xml +++ b/core/metrics-reporter/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-parent - 4.0.11-SNAPSHOT + 4.0.10 ../../microservices/microservice-parent/pom.xml metrics-reporter diff --git a/microservices/services/file-provider/pom.xml b/microservices/services/file-provider/pom.xml index 06499343d28..30a2ccf2c63 100644 --- a/microservices/services/file-provider/pom.xml +++ b/microservices/services/file-provider/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-parent - 4.0.11-SNAPSHOT + 4.0.10 ../../microservice-parent/pom.xml file-provider-service-parent diff --git a/microservices/services/hazelcast/pom.xml b/microservices/services/hazelcast/pom.xml index 6becacd5acc..2f682a6d819 100644 --- a/microservices/services/hazelcast/pom.xml +++ b/microservices/services/hazelcast/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-parent - 4.0.11-SNAPSHOT + 4.0.10 ../../microservice-parent/pom.xml hazelcast-service-parent diff --git a/microservices/services/mapreduce-query/pom.xml b/microservices/services/mapreduce-query/pom.xml index 13bf967f40f..1acf64e05ba 100644 --- a/microservices/services/mapreduce-query/pom.xml +++ b/microservices/services/mapreduce-query/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-parent - 4.0.11-SNAPSHOT + 4.0.10 ../../microservice-parent/pom.xml mapreduce-query-service-parent diff --git a/microservices/services/modification/pom.xml b/microservices/services/modification/pom.xml index 290e3a65e7d..3af4b1f0dcf 100644 --- a/microservices/services/modification/pom.xml +++ b/microservices/services/modification/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-parent - 4.0.11-SNAPSHOT + 4.0.10 ../../microservice-parent/pom.xml modification-service-parent diff --git a/microservices/services/query-executor/api/pom.xml b/microservices/services/query-executor/api/pom.xml index f3032f9ded9..235ed3a37c3 100644 --- a/microservices/services/query-executor/api/pom.xml +++ b/microservices/services/query-executor/api/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-parent - 4.0.11-SNAPSHOT + 4.0.10 ../../../microservice-parent/pom.xml query-executor-api diff --git a/microservices/services/query-executor/pom.xml b/microservices/services/query-executor/pom.xml index 5d030feeb5f..434b7be3d13 100644 --- a/microservices/services/query-executor/pom.xml +++ b/microservices/services/query-executor/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-parent - 4.0.11-SNAPSHOT + 4.0.10 ../../microservice-parent/pom.xml query-executor-service-parent diff --git a/microservices/services/query-executor/service/pom.xml b/microservices/services/query-executor/service/pom.xml index b41a37e8f47..17e8f2df18f 100644 --- a/microservices/services/query-executor/service/pom.xml +++ b/microservices/services/query-executor/service/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-service-parent - 5.0.12-SNAPSHOT + 5.0.11 ../../../microservice-service-parent/pom.xml query-executor-service diff --git a/microservices/services/query-metric/pom.xml b/microservices/services/query-metric/pom.xml index 765d183d4c3..aa1b6ab9bb3 100644 --- a/microservices/services/query-metric/pom.xml +++ b/microservices/services/query-metric/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-parent - 4.0.11-SNAPSHOT + 4.0.10 ../../microservice-parent/pom.xml query-metric-parent diff --git a/microservices/services/query/pom.xml b/microservices/services/query/pom.xml index 369c2076ced..7445827eff0 100644 --- a/microservices/services/query/pom.xml +++ b/microservices/services/query/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-parent - 4.0.11-SNAPSHOT + 4.0.10 ../../microservice-parent/pom.xml query-service-parent diff --git a/microservices/services/query/service/pom.xml b/microservices/services/query/service/pom.xml index 8278519b7ec..9ad2a15fac7 100644 --- a/microservices/services/query/service/pom.xml +++ b/microservices/services/query/service/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-service-parent - 5.0.12-SNAPSHOT + 5.0.11 ../../../microservice-service-parent/pom.xml query-service diff --git a/microservices/starters/cached-results/pom.xml b/microservices/starters/cached-results/pom.xml index ac47212c3c4..3cb0810ac77 100644 --- a/microservices/starters/cached-results/pom.xml +++ b/microservices/starters/cached-results/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-service-parent - 5.0.12-SNAPSHOT + 5.0.11 ../../microservice-service-parent/pom.xml spring-boot-starter-datawave-cached-results From 88d30e7512326450cf7240f2d24f9c13b9ee490e Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Mon, 11 May 2026 14:28:22 -0400 Subject: [PATCH 48/63] add protostuff test cases to AccessExpressionMarkingsTest --- core/utils/accumulo-utils/pom.xml | 5 + .../marking/AccessExpressionMarkingsTest.java | 99 +++++++++++++++++++ 2 files changed, 104 insertions(+) diff --git a/core/utils/accumulo-utils/pom.xml b/core/utils/accumulo-utils/pom.xml index 9f0dc7571ba..527fe855515 100644 --- a/core/utils/accumulo-utils/pom.xml +++ b/core/utils/accumulo-utils/pom.xml @@ -108,6 +108,11 @@ datawave-in-memory-accumulo test + + io.protostuff + protostuff-core + test + org.slf4j slf4j-simple diff --git a/core/utils/accumulo-utils/src/test/java/datawave/marking/AccessExpressionMarkingsTest.java b/core/utils/accumulo-utils/src/test/java/datawave/marking/AccessExpressionMarkingsTest.java index 88b412114f0..c78637b88cd 100644 --- a/core/utils/accumulo-utils/src/test/java/datawave/marking/AccessExpressionMarkingsTest.java +++ b/core/utils/accumulo-utils/src/test/java/datawave/marking/AccessExpressionMarkingsTest.java @@ -4,6 +4,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -14,9 +15,14 @@ import com.fasterxml.jackson.databind.ObjectMapper; +import io.protostuff.LinkedBuffer; +import io.protostuff.ProtostuffIOUtil; +import io.protostuff.Schema; + public class AccessExpressionMarkingsTest { private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final Schema SCHEMA = AccessExpressionMarkings.SCHEMA; // -- getMarkings / toAccessExpression -- @@ -176,4 +182,97 @@ public void testEquality() { assertEquals(a, b); assertEquals(a.hashCode(), b.hashCode()); } + + // -- Protostuff Schema metadata -- + + @Test + public void testSchemaGetFieldNameKnown() { + assertEquals(AccessExpressionMarkings.COLUMN_VISIBILITY_KEY, SCHEMA.getFieldName(1)); + } + + @Test + public void testSchemaGetFieldNameUnknown() { + assertNull(SCHEMA.getFieldName(0)); + assertNull(SCHEMA.getFieldName(2)); + } + + @Test + public void testSchemaGetFieldNumberKnown() { + assertEquals(1, SCHEMA.getFieldNumber(AccessExpressionMarkings.COLUMN_VISIBILITY_KEY)); + } + + @Test + public void testSchemaGetFieldNumberUnknown() { + assertEquals(0, SCHEMA.getFieldNumber("nonExistentField")); + } + + @Test + public void testSchemaIsInitialized() { + assertTrue(SCHEMA.isInitialized(AccessExpressionMarkings.create("A"))); + assertTrue(SCHEMA.isInitialized(AccessExpressionMarkings.builder().build())); + } + + @Test + public void testSchemaNewMessage() { + AccessExpressionMarkings msg = SCHEMA.newMessage(); + assertNotNull(msg); + assertTrue(msg.isEmpty()); + } + + @Test + public void testSchemaMessageName() { + assertEquals("AccessExpressionMarkings", SCHEMA.messageName()); + } + + @Test + public void testSchemaMessageFullName() { + assertEquals("datawave.marking.AccessExpressionMarkings", SCHEMA.messageFullName()); + } + + @Test + public void testSchemaTypeClass() { + assertEquals(AccessExpressionMarkings.class, SCHEMA.typeClass()); + } + + // -- Protostuff Schema round-trip serialization -- + + @Test + public void testProtostuffRoundTrip() { + AccessExpressionMarkings original = AccessExpressionMarkings.create("(A&B)|C"); + + LinkedBuffer buffer = LinkedBuffer.allocate(); + byte[] bytes = ProtostuffIOUtil.toByteArray(original, SCHEMA, buffer); + + AccessExpressionMarkings deserialized = SCHEMA.newMessage(); + ProtostuffIOUtil.mergeFrom(bytes, deserialized, SCHEMA); + + assertEquals(original.getColumnVisibilityString(), deserialized.getColumnVisibilityString()); + } + + @Test + public void testProtostuffRoundTripEmpty() { + AccessExpressionMarkings original = AccessExpressionMarkings.builder().accessExpression(null).build(); + + LinkedBuffer buffer = LinkedBuffer.allocate(); + byte[] bytes = ProtostuffIOUtil.toByteArray(original, SCHEMA, buffer); + + AccessExpressionMarkings deserialized = SCHEMA.newMessage(); + ProtostuffIOUtil.mergeFrom(bytes, deserialized, SCHEMA); + + assertTrue(deserialized.isEmpty()); + } + + @Test + public void testProtostuffRoundTripComplexExpression() { + AccessExpressionMarkings original = AccessExpressionMarkings.create("(A&B)|(C&D&E)"); + + LinkedBuffer buffer = LinkedBuffer.allocate(); + byte[] bytes = ProtostuffIOUtil.toByteArray(original, SCHEMA, buffer); + + AccessExpressionMarkings deserialized = SCHEMA.newMessage(); + ProtostuffIOUtil.mergeFrom(bytes, deserialized, SCHEMA); + + assertEquals("(A&B)|(C&D&E)", deserialized.getColumnVisibilityString()); + assertFalse(deserialized.isEmpty()); + } } From 8b3097bdf6e134622ee71dc7babe76a851675afd Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Mon, 11 May 2026 16:08:16 -0400 Subject: [PATCH 49/63] migrate markingsMap to markings in dictionary api and service --- .../marking/AccessExpressionMarkings.java | 13 ++++++ .../dictionary/data/DefaultDescription.java | 42 ++++++------------- .../dictionary/data/DescriptionBase.java | 24 +---------- .../metadata/DefaultMetadataField.java | 9 ++-- .../data/DefaultDescriptionTest.java | 26 ++++-------- .../DataDictionaryControllerTest.java | 6 ++- .../data/DataDictionaryImplTest.java | 31 ++++---------- .../DefaultMetadataFieldScannerTest.java | 7 +--- 8 files changed, 53 insertions(+), 105 deletions(-) diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java index 5d5f01cd78d..0908cb8b296 100644 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java @@ -4,7 +4,10 @@ import java.nio.charset.StandardCharsets; import java.util.HashMap; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlTransient; +import javax.xml.bind.annotation.XmlValue; import org.apache.accumulo.access.Access; import org.apache.accumulo.access.AccessExpression; @@ -18,11 +21,16 @@ import io.protostuff.Input; import io.protostuff.Output; import io.protostuff.Schema; +import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; +import lombok.NoArgsConstructor; @Data @Builder +@NoArgsConstructor +@AllArgsConstructor +@XmlAccessorType(XmlAccessType.NONE) @JsonAutoDetect(getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE) public class AccessExpressionMarkings implements Markings { public static final Access ACCESS = Access.builder().build(); @@ -52,10 +60,15 @@ public boolean isEmpty() { } @JsonProperty(COLUMN_VISIBILITY_KEY) + @XmlValue public String getColumnVisibilityString() { return accessExpression != null ? accessExpression.getExpression() : ""; } + public void setColumnVisibilityString(String expr) { + this.accessExpression = ACCESS.newExpression(expr != null ? expr : ""); + } + @JsonCreator public static AccessExpressionMarkings create(@JsonProperty(COLUMN_VISIBILITY_KEY) String ae) { return AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression(ae != null ? ae : "")).build(); diff --git a/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java b/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java index 4f559bacc6c..3b4f1f8bb47 100644 --- a/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java +++ b/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java @@ -1,24 +1,18 @@ package datawave.webservice.dictionary.data; -import static datawave.marking.AccessExpressionMarkings.ACCESS; - import java.io.IOException; import java.io.Serializable; -import java.util.HashMap; -import java.util.Map; import java.util.Objects; import javax.xml.bind.annotation.XmlAccessOrder; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorOrder; import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlTransient; -import org.apache.accumulo.access.AccessExpression; - import datawave.marking.AccessExpressionMarkings; import datawave.marking.Markings; -import datawave.webservice.query.result.event.MapSchema; import io.protostuff.Input; import io.protostuff.Message; import io.protostuff.Output; @@ -29,19 +23,18 @@ public class DefaultDescription extends DescriptionBase implements Serializable, Message { private static final long serialVersionUID = 6756537335803261872L; - private static final String CV_KEY = "columnVisibility"; - public DefaultDescription() {} public DefaultDescription(String description) { this.description = description; } - public DefaultDescription(String description, Map markingsMap) { + public DefaultDescription(String description, Markings markings) { this.description = description; - this.markingsMap = markingsMap; + this.markings = markings; } + @XmlElement(name = "description") public String getDescription() { return description; } @@ -51,23 +44,14 @@ public void setDescription(String description) { } @Override - public void setMarkings(Markings markings) { - if (markings == null || markings.isEmpty()) { - this.markingsMap = null; - } else { - AccessExpression ae = markings.toAccessExpression(); - this.markingsMap = new HashMap<>(); - this.markingsMap.put(CV_KEY, ae.getExpression()); - } + @XmlElement(name = "markings", type = AccessExpressionMarkings.class) + public Markings getMarkings() { + return this.markings; } @Override - public Markings getMarkings() { - if (this.markingsMap != null && this.markingsMap.containsKey(CV_KEY)) { - String expr = this.markingsMap.get(CV_KEY); - return AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression(expr)).build(); - } - return null; + public void setMarkings(Markings markings) { + this.markings = markings; } @Override @@ -130,8 +114,8 @@ public void writeTo(Output output, DefaultDescription message) throws IOExceptio if (message.description != null) { output.writeString(1, message.description, false); } - if (message.markingsMap != null) - output.writeObject(1, message.markingsMap, MapSchema.SCHEMA, false); + if (message.markings != null) + output.writeObject(2, (AccessExpressionMarkings) message.markings, AccessExpressionMarkings.SCHEMA, false); } public void mergeFrom(Input input, DefaultDescription message) throws IOException { @@ -142,8 +126,8 @@ public void mergeFrom(Input input, DefaultDescription message) throws IOExceptio message.description = input.readString(); break; case 2: - message.markingsMap = new HashMap<>(); - input.mergeObject(message.markingsMap, MapSchema.SCHEMA); + message.markings = AccessExpressionMarkings.builder().build(); + input.mergeObject((AccessExpressionMarkings) message.markings, AccessExpressionMarkings.SCHEMA); break; default: input.handleUnknownField(number, this); diff --git a/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java b/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java index 10c181cfd33..0e22cc1fde0 100644 --- a/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java +++ b/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java @@ -1,34 +1,14 @@ package datawave.webservice.dictionary.data; -import java.util.Map; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; - +import datawave.marking.Markings; import datawave.webservice.query.result.event.HasMarkings; -import datawave.webservice.xml.util.StringMapAdapter; import io.protostuff.Message; -import lombok.Getter; -import lombok.Setter; -@XmlAccessorType(XmlAccessType.NONE) public abstract class DescriptionBase implements HasMarkings, Message { - @XmlElement(name = "description") protected String description; - /** - * JAXB-serialized markings map. The {@link HasMarkings} interface methods {@link #getMarkings()}/{@link #setMarkings} operate on {@code Markings}; - * subclasses must bridge between this map and the Markings type. -- GETTER -- Direct access to the JAXB markings map. -- SETTER -- Direct access to set the - * JAXB markings map. - */ - @Setter - @Getter - @XmlElement(name = "markings") - @XmlJavaTypeAdapter(StringMapAdapter.class) - protected Map markingsMap; + protected Markings markings; public abstract String getDescription(); diff --git a/microservices/services/dictionary/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java b/microservices/services/dictionary/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java index e4fd23b0fcd..0d93ab22039 100644 --- a/microservices/services/dictionary/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java +++ b/microservices/services/dictionary/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java @@ -7,7 +7,6 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; import javax.xml.bind.annotation.XmlAccessType; @@ -21,9 +20,9 @@ import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; +import datawave.marking.AccessExpressionMarkings; import datawave.webservice.dictionary.data.DefaultDescription; import datawave.webservice.dictionary.data.DescriptionBase; -import datawave.webservice.query.result.event.MapSchema; import io.protostuff.Input; import io.protostuff.Message; import io.protostuff.Output; @@ -250,8 +249,8 @@ public void mergeFrom(Input input, DefaultMetadataField message) throws IOExcept int size = input.readInt32(); message.descriptions = new HashSet<>(size); for (int i = 0; i < size; i++) { - Map markings = new HashMap<>(); - input.mergeObject(markings, MapSchema.SCHEMA); + AccessExpressionMarkings markings = AccessExpressionMarkings.builder().build(); + input.mergeObject(markings, AccessExpressionMarkings.SCHEMA); message.descriptions.add(new DefaultDescription(input.readString(), markings)); } break; @@ -295,7 +294,7 @@ public void writeTo(Output output, DefaultMetadataField message) throws IOExcept output.writeInt32(9, message.getDescriptions().size(), false); for (DescriptionBase desc : message.getDescriptions()) { output.writeString(9, desc.getDescription(), true); - output.writeObject(9, desc.getMarkingsMap(), MapSchema.SCHEMA, false); + output.writeObject(9, (AccessExpressionMarkings) desc.getMarkings(), AccessExpressionMarkings.SCHEMA, false); } output.writeString(10, message.lastUpdated, false); output.writeBool(11, message.tokenized, false); diff --git a/microservices/services/dictionary/api/src/test/java/datawave/webservice/dictionary/data/DefaultDescriptionTest.java b/microservices/services/dictionary/api/src/test/java/datawave/webservice/dictionary/data/DefaultDescriptionTest.java index fbb98bc2e4a..8fa8e8dc145 100644 --- a/microservices/services/dictionary/api/src/test/java/datawave/webservice/dictionary/data/DefaultDescriptionTest.java +++ b/microservices/services/dictionary/api/src/test/java/datawave/webservice/dictionary/data/DefaultDescriptionTest.java @@ -7,10 +7,8 @@ import java.io.ByteArrayOutputStream; import java.lang.annotation.Annotation; import java.util.ArrayList; -import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; import javax.xml.bind.JAXBContext; @@ -21,6 +19,8 @@ import org.junit.jupiter.api.Test; +import datawave.marking.AccessExpressionMarkings; + public class DefaultDescriptionTest { @Test @@ -37,17 +37,13 @@ public void testMarshall() throws JAXBException { String result = out.toString(); - assertTrue(result.contains("columnVisibility")); - assertTrue(result.contains("PRIVATE")); + assertTrue(result.contains("PRIVATE")); assertTrue(result.contains("my description")); } private static DefaultFields getDefaultFields() { - Map markings = new HashMap<>(); - markings.put("columnVisibility", "PRIVATE"); - DefaultDescription desc = new DefaultDescription(); - desc.setMarkingsMap(markings); + desc.setMarkings(AccessExpressionMarkings.create("PRIVATE")); desc.setDescription("my description"); Set descs = new HashSet<>(); @@ -87,12 +83,7 @@ public void testUnmarshall() throws JAXBException { + " myType\n" + " \n" + " my description\n" - + " \n" - + " \n" - + " columnVisibility\n" - + " PRIVATE\n" - + " \n" - + " \n" + + " PRIVATE\n" + " \n" + " myField\n" + " \n" @@ -112,9 +103,9 @@ public void testUnmarshall() throws JAXBException { assertEquals("myType", f.getDatatype()); for (DefaultDescription d : f.getDescriptions()) { assertEquals("my description", d.getDescription()); - Map m = d.getMarkingsMap(); - assertEquals("PRIVATE", m.get("columnVisibility")); - markingCount = m.size(); + assertTrue(d.getMarkings() instanceof AccessExpressionMarkings); + AccessExpressionMarkings aem = (AccessExpressionMarkings) d.getMarkings(); + assertEquals("PRIVATE", aem.getColumnVisibilityString()); descriptionCount++; } fieldCount++; @@ -122,6 +113,5 @@ public void testUnmarshall() throws JAXBException { assertEquals(1, fieldCount); assertEquals(1, descriptionCount); - assertEquals(1, markingCount); } } diff --git a/microservices/services/dictionary/service/src/test/java/datawave/microservice/dictionary/DataDictionaryControllerTest.java b/microservices/services/dictionary/service/src/test/java/datawave/microservice/dictionary/DataDictionaryControllerTest.java index 1e5512f7840..aede610bfaf 100644 --- a/microservices/services/dictionary/service/src/test/java/datawave/microservice/dictionary/DataDictionaryControllerTest.java +++ b/microservices/services/dictionary/service/src/test/java/datawave/microservice/dictionary/DataDictionaryControllerTest.java @@ -31,6 +31,7 @@ import datawave.accumulo.inmemory.InMemoryAccumuloClient; import datawave.accumulo.inmemory.InMemoryInstance; +import datawave.marking.AccessExpressionMarkings; import datawave.microservice.ControllerIT; import datawave.microservice.dictionary.config.DataDictionaryProperties; import datawave.webservice.dictionary.data.DefaultDataDictionary; @@ -128,8 +129,9 @@ public void testPostDescriptions() { // @formatter:on Multimap,DefaultDescription> descriptions = HashMultimap.create(); - descriptions.put(new AbstractMap.SimpleEntry<>("fooField", "fooType"), new DefaultDescription("my foo field", Map.of("columnVisibility", "USER"))); - descriptions.put(new AbstractMap.SimpleEntry<>("barField", "barType"), new DefaultDescription("my bar field", Map.of("columnVisibility", "ADMIN"))); + descriptions.put(new AbstractMap.SimpleEntry<>("fooField", "fooType"), new DefaultDescription("my foo field", AccessExpressionMarkings.create("USER"))); + descriptions.put(new AbstractMap.SimpleEntry<>("barField", "barType"), + new DefaultDescription("my bar field", AccessExpressionMarkings.create("ADMIN"))); DefaultFields postBody = new DefaultFields(descriptions); MultiValueMap additionalHeaders = new LinkedMultiValueMap<>(); additionalHeaders.set(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE); diff --git a/microservices/services/dictionary/service/src/test/java/datawave/microservice/dictionary/data/DataDictionaryImplTest.java b/microservices/services/dictionary/service/src/test/java/datawave/microservice/dictionary/data/DataDictionaryImplTest.java index 6613613cb0d..20254d103ce 100644 --- a/microservices/services/dictionary/service/src/test/java/datawave/microservice/dictionary/data/DataDictionaryImplTest.java +++ b/microservices/services/dictionary/service/src/test/java/datawave/microservice/dictionary/data/DataDictionaryImplTest.java @@ -28,6 +28,7 @@ import com.google.common.collect.Multimap; import com.google.common.collect.SetMultimap; +import datawave.marking.AccessExpressionMarkings; import datawave.marking.MarkingFunctions; import datawave.microservice.Connection; import datawave.microservice.dictionary.config.ResponseObjectFactory; @@ -88,11 +89,8 @@ public void setUp() { @Test public void whenSettingDescription_givenSingleDefaultDescription_shouldSetDescription() throws Exception { - Map markings = new HashMap<>(); - markings.put("columnVisibility", "PRIVATE"); - DefaultDescription description = new DefaultDescription(); - description.setMarkingsMap(markings); + description.setMarkings(AccessExpressionMarkings.create("PRIVATE")); description.setDescription("my ultra cool description"); // Ensure no alias will be found. @@ -110,11 +108,8 @@ public void whenSettingDescription_givenSingleDefaultDescription_shouldSetDescri @Test public void whenSettingDescription_givenDefaultDictionaryField_shouldSetDescriptionFromProperties() throws Exception { - Map markings = new HashMap<>(); - markings.put("columnVisibility", "PRIVATE"); - DefaultDescription description = new DefaultDescription(); - description.setMarkingsMap(markings); + description.setMarkings(AccessExpressionMarkings.create("PRIVATE")); description.setDescription("my ultra cool description"); Set descriptions = Collections.singleton(description); @@ -138,11 +133,8 @@ public void whenSettingDescription_givenDefaultDictionaryField_shouldSetDescript @Test public void whenSettingDescription_givenNoAlias_shouldSetDescriptionWithOriginalFieldName() throws Exception { - Map markings = new HashMap<>(); - markings.put("columnVisibility", "PRIVATE"); - DefaultDescription description = new DefaultDescription(); - description.setMarkingsMap(markings); + description.setMarkings(AccessExpressionMarkings.create("PRIVATE")); description.setDescription("my ultra cool description"); Set descriptions = Collections.singleton(description); @@ -161,11 +153,8 @@ public void whenSettingDescription_givenNoAlias_shouldSetDescriptionWithOriginal @Test public void whenSettingDescription_givenAlias_shouldSetDescriptionWithAlias() throws Exception { - Map markings = new HashMap<>(); - markings.put("columnVisibility", "PRIVATE"); - DefaultDescription description = new DefaultDescription(); - description.setMarkingsMap(markings); + description.setMarkings(AccessExpressionMarkings.create("PRIVATE")); description.setDescription("my ultra cool description"); Set descriptions = Collections.singleton(description); @@ -294,11 +283,8 @@ public void whenRetrievingDescriptionsWithFieldNameAndDatatype_givenAlias_should @Test public void whenDeletingDescription_givenNoAlias_shouldDeleteDescriptionWithOriginalFieldName() throws Exception { - Map markings = new HashMap<>(); - markings.put("columnVisibility", "PRIVATE"); - DefaultDescription description = new DefaultDescription(); - description.setMarkingsMap(markings); + description.setMarkings(AccessExpressionMarkings.create("PRIVATE")); description.setDescription("my ultra cool description"); // Ensure no alias will be found. @@ -315,11 +301,8 @@ public void whenDeletingDescription_givenNoAlias_shouldDeleteDescriptionWithOrig @Test public void whenDeletingDescription_givenAlias_shouldDeleteDescriptionWithAlias() throws Exception { - Map markings = new HashMap<>(); - markings.put("columnVisibility", "PRIVATE"); - DefaultDescription description = new DefaultDescription(); - description.setMarkingsMap(markings); + description.setMarkings(AccessExpressionMarkings.create("PRIVATE")); description.setDescription("my ultra cool description"); // Ensure an alias will be found. diff --git a/microservices/services/dictionary/service/src/test/java/datawave/microservice/metadata/DefaultMetadataFieldScannerTest.java b/microservices/services/dictionary/service/src/test/java/datawave/microservice/metadata/DefaultMetadataFieldScannerTest.java index c9af1e44b60..e6c1fffe1b6 100644 --- a/microservices/services/dictionary/service/src/test/java/datawave/microservice/metadata/DefaultMetadataFieldScannerTest.java +++ b/microservices/services/dictionary/service/src/test/java/datawave/microservice/metadata/DefaultMetadataFieldScannerTest.java @@ -31,6 +31,7 @@ import datawave.accumulo.inmemory.InMemoryAccumuloClient; import datawave.accumulo.inmemory.InMemoryInstance; import datawave.data.ColumnFamilyConstants; +import datawave.marking.AccessExpressionMarkings; import datawave.marking.MarkingFunctions; import datawave.microservice.Connection; import datawave.microservice.dictionary.config.ResponseObjectFactory; @@ -276,11 +277,7 @@ private void populateMetadataTable() throws TableNotFoundException, MutationsRej private DefaultDescription createDescription(String descriptionText) { DefaultDescription description = new DefaultDescription(); description.setDescription(descriptionText); - - Map markings = new HashMap<>(); - markings.put("columnVisibility", "PRIVATE}"); - description.setMarkingsMap(markings); - + description.setMarkings(AccessExpressionMarkings.create("PRIVATE")); return description; } } From da2af39e7d64b2b0b624d2141aad68ce750a4a50 Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Mon, 11 May 2026 17:08:18 -0400 Subject: [PATCH 50/63] more MR fixes --- core/utils/accumulo-utils/pom.xml | 59 +++++++++++ .../marking/AccessExpressionMarkings.java | 4 +- .../datawave/marking/package-info.java | 6 ++ .../marking/AccessExpressionMarkingsTest.java | 97 +++++++++++++++++++ .../dictionary/data/DescriptionBase.java | 4 + .../data/DefaultDescriptionTest.java | 6 +- .../query/tld/TLDFieldIndexAggregator.java | 4 +- .../transformer/DocumentTransformer.java | 9 +- .../FieldIndexCountQueryTransformer.java | 3 +- .../query/result/event/SimpleEvent.java | 18 +--- .../query/result/event/SimpleField.java | 15 +-- .../common/grouping/DocumentGrouperTest.java | 1 - .../query/result/event/DefaultEvent.java | 16 +++ .../query/result/event/DefaultField.java | 4 + .../query/result/event/EventBase.java | 4 - .../query/result/event/FieldBase.java | 2 - .../result/keyword/DefaultTagCloud.java | 15 +++ 17 files changed, 220 insertions(+), 47 deletions(-) create mode 100644 core/utils/accumulo-utils/src/main/resources/source-templates/datawave/marking/package-info.java diff --git a/core/utils/accumulo-utils/pom.xml b/core/utils/accumulo-utils/pom.xml index 527fe855515..0d015ab8147 100644 --- a/core/utils/accumulo-utils/pom.xml +++ b/core/utils/accumulo-utils/pom.xml @@ -9,6 +9,7 @@ accumulo-utils ${project.artifactId} + http://webservice.datawave.nsa/v1 4.1.2 1.1.7 @@ -103,6 +104,12 @@ spring-jcl runtime + + com.sun.xml.bind + jaxb-impl + ${version.jaxb-impl} + test + gov.nsa.datawave datawave-in-memory-accumulo @@ -131,4 +138,56 @@ https://maven.pkg.github.com/NationalSecurityAgency/datawave + + + + true + src/main/resources + + source-templates/** + + + + + + maven-resources-plugin + + + copy-templated-sources + + copy-resources + + validate + + ${project.build.directory}/generated-sources/templated-sources + + + src/main/resources/source-templates + true + + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-source + + add-source + + generate-sources + + + target/generated-sources/templated-sources + + + + + + + diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java index 0908cb8b296..7d0f69ef4c5 100644 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java @@ -6,8 +6,8 @@ import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlTransient; -import javax.xml.bind.annotation.XmlValue; import org.apache.accumulo.access.Access; import org.apache.accumulo.access.AccessExpression; @@ -60,7 +60,7 @@ public boolean isEmpty() { } @JsonProperty(COLUMN_VISIBILITY_KEY) - @XmlValue + @XmlElement(name = COLUMN_VISIBILITY_KEY) public String getColumnVisibilityString() { return accessExpression != null ? accessExpression.getExpression() : ""; } diff --git a/core/utils/accumulo-utils/src/main/resources/source-templates/datawave/marking/package-info.java b/core/utils/accumulo-utils/src/main/resources/source-templates/datawave/marking/package-info.java new file mode 100644 index 00000000000..6a47ac1a52a --- /dev/null +++ b/core/utils/accumulo-utils/src/main/resources/source-templates/datawave/marking/package-info.java @@ -0,0 +1,6 @@ +@XmlSchema(namespace="${datawave.webservice.namespace}", elementFormDefault=XmlNsForm.QUALIFIED, xmlns={@XmlNs(prefix = "", namespaceURI = "${datawave.webservice.namespace}")}) +package datawave.marking; + +import javax.xml.bind.annotation.XmlNs; +import javax.xml.bind.annotation.XmlNsForm; +import javax.xml.bind.annotation.XmlSchema; diff --git a/core/utils/accumulo-utils/src/test/java/datawave/marking/AccessExpressionMarkingsTest.java b/core/utils/accumulo-utils/src/test/java/datawave/marking/AccessExpressionMarkingsTest.java index c78637b88cd..17f5c2d7869 100644 --- a/core/utils/accumulo-utils/src/test/java/datawave/marking/AccessExpressionMarkingsTest.java +++ b/core/utils/accumulo-utils/src/test/java/datawave/marking/AccessExpressionMarkingsTest.java @@ -8,6 +8,15 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBElement; +import javax.xml.bind.Marshaller; +import javax.xml.bind.Unmarshaller; +import javax.xml.namespace.QName; + import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.hadoop.io.Text; @@ -183,6 +192,94 @@ public void testEquality() { assertEquals(a.hashCode(), b.hashCode()); } + // -- JAXB round-trip -- + + @Test + public void testJaxbMarshal() throws Exception { + JAXBContext ctx = JAXBContext.newInstance(AccessExpressionMarkings.class); + Marshaller marshaller = ctx.createMarshaller(); + + AccessExpressionMarkings original = AccessExpressionMarkings.create("A&B"); + JAXBElement element = new JAXBElement<>(new QName("http://webservice.datawave.nsa/v1", "markings"), + AccessExpressionMarkings.class, original); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + marshaller.marshal(element, out); + String xml = out.toString(); + + assertTrue(xml.contains("A&B"), "XML should contain the columnVisibility element: " + xml); + } + + @Test + public void testJaxbUnmarshal() throws Exception { + JAXBContext ctx = JAXBContext.newInstance(AccessExpressionMarkings.class); + Unmarshaller unmarshaller = ctx.createUnmarshaller(); + + String xml = "" + + "X|Y"; + JAXBElement element = unmarshaller + .unmarshal(new javax.xml.transform.stream.StreamSource(new ByteArrayInputStream(xml.getBytes())), AccessExpressionMarkings.class); + AccessExpressionMarkings markings = element.getValue(); + + assertEquals("X|Y", markings.getColumnVisibilityString()); + assertFalse(markings.isEmpty()); + } + + @Test + public void testJaxbRoundTrip() throws Exception { + JAXBContext ctx = JAXBContext.newInstance(AccessExpressionMarkings.class); + Marshaller marshaller = ctx.createMarshaller(); + Unmarshaller unmarshaller = ctx.createUnmarshaller(); + + AccessExpressionMarkings original = AccessExpressionMarkings.create("(A&B)|C"); + JAXBElement element = new JAXBElement<>(new QName("http://webservice.datawave.nsa/v1", "markings"), + AccessExpressionMarkings.class, original); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + marshaller.marshal(element, out); + + JAXBElement result = unmarshaller + .unmarshal(new javax.xml.transform.stream.StreamSource(new ByteArrayInputStream(out.toByteArray())), AccessExpressionMarkings.class); + assertEquals(original.getColumnVisibilityString(), result.getValue().getColumnVisibilityString()); + } + + @Test + public void testJaxbRoundTripEmpty() throws Exception { + JAXBContext ctx = JAXBContext.newInstance(AccessExpressionMarkings.class); + Marshaller marshaller = ctx.createMarshaller(); + Unmarshaller unmarshaller = ctx.createUnmarshaller(); + + AccessExpressionMarkings original = AccessExpressionMarkings.create(""); + JAXBElement element = new JAXBElement<>(new QName("http://webservice.datawave.nsa/v1", "markings"), + AccessExpressionMarkings.class, original); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + marshaller.marshal(element, out); + + JAXBElement result = unmarshaller + .unmarshal(new javax.xml.transform.stream.StreamSource(new ByteArrayInputStream(out.toByteArray())), AccessExpressionMarkings.class); + assertEquals("", result.getValue().getColumnVisibilityString()); + assertTrue(result.getValue().isEmpty()); + } + + @Test + public void testJaxbRoundTripComplexExpression() throws Exception { + JAXBContext ctx = JAXBContext.newInstance(AccessExpressionMarkings.class); + Marshaller marshaller = ctx.createMarshaller(); + Unmarshaller unmarshaller = ctx.createUnmarshaller(); + + AccessExpressionMarkings original = AccessExpressionMarkings.create("(A&B)|(C&D&E)"); + JAXBElement element = new JAXBElement<>(new QName("http://webservice.datawave.nsa/v1", "markings"), + AccessExpressionMarkings.class, original); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + marshaller.marshal(element, out); + + JAXBElement result = unmarshaller + .unmarshal(new javax.xml.transform.stream.StreamSource(new ByteArrayInputStream(out.toByteArray())), AccessExpressionMarkings.class); + assertEquals("(A&B)|(C&D&E)", result.getValue().getColumnVisibilityString()); + assertFalse(result.getValue().isEmpty()); + } // -- Protostuff Schema metadata -- @Test diff --git a/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java b/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java index 0e22cc1fde0..777cfd02c01 100644 --- a/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java +++ b/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java @@ -1,9 +1,13 @@ package datawave.webservice.dictionary.data; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + import datawave.marking.Markings; import datawave.webservice.query.result.event.HasMarkings; import io.protostuff.Message; +@XmlAccessorType(XmlAccessType.NONE) public abstract class DescriptionBase implements HasMarkings, Message { protected String description; diff --git a/microservices/services/dictionary/api/src/test/java/datawave/webservice/dictionary/data/DefaultDescriptionTest.java b/microservices/services/dictionary/api/src/test/java/datawave/webservice/dictionary/data/DefaultDescriptionTest.java index 8fa8e8dc145..181028e620e 100644 --- a/microservices/services/dictionary/api/src/test/java/datawave/webservice/dictionary/data/DefaultDescriptionTest.java +++ b/microservices/services/dictionary/api/src/test/java/datawave/webservice/dictionary/data/DefaultDescriptionTest.java @@ -37,7 +37,7 @@ public void testMarshall() throws JAXBException { String result = out.toString(); - assertTrue(result.contains("PRIVATE")); + assertTrue(result.contains("PRIVATE")); assertTrue(result.contains("my description")); } @@ -83,7 +83,9 @@ public void testUnmarshall() throws JAXBException { + " myType\n" + " \n" + " my description\n" - + " PRIVATE\n" + + " \n" + + " PRIVATE\n" + + " \n" + " \n" + " myField\n" + " \n" diff --git a/warehouse/query-core/src/main/java/datawave/query/tld/TLDFieldIndexAggregator.java b/warehouse/query-core/src/main/java/datawave/query/tld/TLDFieldIndexAggregator.java index 5031c70acc3..06a7738edb6 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tld/TLDFieldIndexAggregator.java +++ b/warehouse/query-core/src/main/java/datawave/query/tld/TLDFieldIndexAggregator.java @@ -12,6 +12,7 @@ import org.apache.accumulo.core.iterators.SortedKeyValueIterator; import org.apache.hadoop.io.Text; +import datawave.marking.ColumnVisibilityCache; import datawave.query.Constants; import datawave.query.attributes.Attribute; import datawave.query.attributes.AttributeFactory; @@ -57,7 +58,8 @@ public Key apply(SortedKeyValueIterator itr, Document d, AttributeFac ByteSequence thisId = parseDatatypeUidFromFI(key.getColumnQualifierData()); if (docId == null || !docId.equals(thisId)) { docId = thisId; - Key docKey = new Key(key.getRow(), new Text(docId.toArray()), new Text(), key.getColumnVisibilityParsed(), key.getTimestamp()); + Key docKey = new Key(key.getRow(), new Text(docId.toArray()), new Text(), ColumnVisibilityCache.get(key.getColumnVisibilityData()), + key.getTimestamp()); attr = new DocumentKey(docKey, false); d.put(Document.DOCKEY_FIELD_NAME, attr); } diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java index 2331b11a278..d4f2c17f377 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java @@ -163,14 +163,9 @@ private EventBase _transform(Entry documentEntry) throws EmptyObje } protected EventBase buildResponse(Document document, Key documentKey, ColumnVisibility eventCV, String colf, String row, - MarkingFunctions markingFunctions) { + MarkingFunctions markingFunctions) throws MarkingFunctions.Exception { - Markings markings; - try { - markings = markingFunctions.translateFromColumnVisibility(eventCV); - } catch (MarkingFunctions.Exception e) { - throw new RuntimeException(e); - } + Markings markings = markingFunctions.translateFromColumnVisibility(eventCV); EventBase event = null; final Collection> documentFields = buildDocumentFields(documentKey, null, document, eventCV, markingFunctions); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/FieldIndexCountQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/FieldIndexCountQueryTransformer.java index 4338887e6cc..28ee4430bf8 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/FieldIndexCountQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/FieldIndexCountQueryTransformer.java @@ -64,7 +64,8 @@ public EventBase transform(Entry entry) { try { markings = markingFunctions.translateFromColumnVisibilityForAuths(val.getColumnVisibility(), this.auths); } catch (MarkingFunctions.Exception e) { - throw new RuntimeException(e); + log.error("could not translate " + val.getColumnVisibility() + " to markings, skipping entry"); + return null; } if (null == markings || markings.isEmpty()) { // can't process this one because we did not have valid security markings diff --git a/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleEvent.java b/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleEvent.java index 4dfe7231f12..2f0c9848edb 100644 --- a/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleEvent.java +++ b/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleEvent.java @@ -1,7 +1,5 @@ package datawave.webservice.query.result.event; -import static datawave.marking.AccessExpressionMarkings.ACCESS; - import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; @@ -16,8 +14,6 @@ import javax.xml.bind.annotation.XmlElementWrapper; import javax.xml.bind.annotation.XmlTransient; -import org.apache.accumulo.access.AccessExpression; - import datawave.marking.AccessExpressionMarkings; import datawave.marking.Markings; import datawave.webservice.query.data.ObjectSizeOf; @@ -106,11 +102,7 @@ public void writeTo(Output output, SimpleEvent message) throws IOException { } } if (message.markings != null) { - // Serialize markings as the expression string for backward compatibility - AccessExpression ae = message.markings.toAccessExpression(); - if (ae != null && ae.getExpression() != null) { - output.writeString(3, ae.getExpression(), false); - } + output.writeObject(3, (AccessExpressionMarkings) message.markings, AccessExpressionMarkings.SCHEMA, false); } } @@ -128,10 +120,8 @@ public void mergeFrom(Input input, SimpleEvent message) throws IOException { message.fields.add(f); break; case 3: - String expr = input.readString(); - if (expr != null && !expr.isEmpty()) { - message.markings = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression(expr)).build(); - } + message.markings = AccessExpressionMarkings.builder().build(); + input.mergeObject((AccessExpressionMarkings) message.markings, AccessExpressionMarkings.SCHEMA); break; default: input.handleUnknownField(number, this); @@ -148,8 +138,6 @@ public String getFieldName(int number) { return "fields"; case 3: return "markings"; - case 4: - return "payload"; default: return null; } diff --git a/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleField.java b/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleField.java index d0971a9ed30..cffb337c7a1 100644 --- a/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleField.java +++ b/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleField.java @@ -1,7 +1,5 @@ package datawave.webservice.query.result.event; -import static datawave.marking.AccessExpressionMarkings.ACCESS; - import java.io.IOException; import java.io.Serializable; import java.util.HashMap; @@ -16,7 +14,6 @@ import javax.xml.bind.annotation.XmlElementWrapper; import javax.xml.bind.annotation.XmlTransient; -import org.apache.accumulo.access.AccessExpression; import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; @@ -197,11 +194,7 @@ public void writeTo(Output output, SimpleField message) throws IOException { if (message.value != null) output.writeObject(4, message.value, message.value.cachedSchema(), false); if (message.markings != null) { - // Serialize markings as the expression string for backward compatibility - AccessExpression ae = message.markings.toAccessExpression(); - if (ae != null && ae.getExpression() != null) { - output.writeString(5, ae.getExpression(), false); - } + output.writeObject(5, (AccessExpressionMarkings) message.markings, AccessExpressionMarkings.SCHEMA, false); } } @@ -223,10 +216,8 @@ public void mergeFrom(Input input, SimpleField message) throws IOException { message.value = input.mergeObject(null, TypedValue.getSchema()); break; case 5: - String expr = input.readString(); - if (expr != null && !expr.isEmpty()) { - message.markings = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression(expr)).build(); - } + message.markings = AccessExpressionMarkings.builder().build(); + input.mergeObject((AccessExpressionMarkings) message.markings, AccessExpressionMarkings.SCHEMA); break; default: input.handleUnknownField(number, this); diff --git a/warehouse/query-core/src/test/java/datawave/query/common/grouping/DocumentGrouperTest.java b/warehouse/query-core/src/test/java/datawave/query/common/grouping/DocumentGrouperTest.java index 4e213de1e5f..f695791d713 100644 --- a/warehouse/query-core/src/test/java/datawave/query/common/grouping/DocumentGrouperTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/common/grouping/DocumentGrouperTest.java @@ -37,7 +37,6 @@ public class DocumentGrouperTest { private static final ColumnVisibility COLVIS_E = new ColumnVisibility("E"); private static final ColumnVisibility COLVIS_I = new ColumnVisibility("I"); private static final ColumnVisibility COLVIS_ALL_E_I = new ColumnVisibility("ALL&E&I"); - private static final Key key = new Key("test_key"); private static final Multimap inverseReverseMap = HashMultimap.create(); private static final Map reverseMap = new HashMap<>(); diff --git a/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultEvent.java b/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultEvent.java index 1d1b280850f..cee73ef751d 100644 --- a/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultEvent.java +++ b/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultEvent.java @@ -17,6 +17,7 @@ import org.apache.commons.lang.builder.EqualsBuilder; import datawave.marking.AccessExpressionMarkings; +import datawave.marking.Markings; import datawave.webservice.query.data.ObjectSizeOf; import io.protostuff.Input; import io.protostuff.Message; @@ -29,6 +30,9 @@ public class DefaultEvent extends EventBase implement private static final long serialVersionUID = 2390592143914560317L; + @XmlElement(name = "Markings") + private Markings markings; + @XmlElement(name = "Metadata") private Metadata metadata; @@ -49,6 +53,17 @@ public String toString() { return getMarkings() + ": " + (this.fields != null ? this.fields.toString() : "fields are null"); } + @Override + @XmlElement(name = "Markings", type = AccessExpressionMarkings.class) + public Markings getMarkings() { + return this.markings; + } + + @Override + public void setMarkings(Markings markings) { + this.markings = markings; + } + public Metadata getMetadata() { return metadata; } @@ -135,6 +150,7 @@ public void mergeFrom(Input input, DefaultEvent message) throws IOException { while ((number = input.readFieldNumber(this)) != 0) { switch (number) { case 1: + message.markings = AccessExpressionMarkings.builder().build(); input.mergeObject((AccessExpressionMarkings) message.markings, AccessExpressionMarkings.SCHEMA); break; case 2: diff --git a/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultField.java b/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultField.java index cd8277cc3ff..21c66cde6bd 100644 --- a/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultField.java +++ b/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultField.java @@ -37,6 +37,8 @@ public class DefaultField extends FieldBase implements Serializabl private static final long serialVersionUID = -3982566563059126017L; + @XmlElement(name = "Markings") + private Markings markings; @XmlAttribute(name = "columnVisibility") private String columnVisibility; @XmlAttribute(name = "timestamp") @@ -70,6 +72,7 @@ public void setMarkings(Markings markings) { this.markings = markings; } + @Override public Markings getMarkings() { return markings; } @@ -223,6 +226,7 @@ public void mergeFrom(Input input, DefaultField message) throws IOException { while ((number = input.readFieldNumber(this)) != 0) { switch (number) { case 1: + message.markings = AccessExpressionMarkings.builder().build(); input.mergeObject((AccessExpressionMarkings) message.markings, AccessExpressionMarkings.SCHEMA); break; case 2: diff --git a/web-services/client/src/main/java/datawave/webservice/query/result/event/EventBase.java b/web-services/client/src/main/java/datawave/webservice/query/result/event/EventBase.java index 3eb5668ef22..ce367657cd0 100644 --- a/web-services/client/src/main/java/datawave/webservice/query/result/event/EventBase.java +++ b/web-services/client/src/main/java/datawave/webservice/query/result/event/EventBase.java @@ -4,14 +4,11 @@ import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlSeeAlso; -import javax.xml.bind.annotation.XmlTransient; import datawave.marking.Markings; import io.protostuff.Message; @XmlAccessorType(XmlAccessType.NONE) -@XmlSeeAlso(DefaultEvent.class) public abstract class EventBase> implements HasMarkings, Message { protected transient Markings markings; @@ -45,7 +42,6 @@ public abstract class EventBase> implements HasMarkings public abstract void setFields(List fields); - @XmlTransient public abstract List getFields(); public Markings getMarkings() { diff --git a/web-services/client/src/main/java/datawave/webservice/query/result/event/FieldBase.java b/web-services/client/src/main/java/datawave/webservice/query/result/event/FieldBase.java index 30c7b8069d6..f8883f6dcd0 100644 --- a/web-services/client/src/main/java/datawave/webservice/query/result/event/FieldBase.java +++ b/web-services/client/src/main/java/datawave/webservice/query/result/event/FieldBase.java @@ -4,7 +4,6 @@ import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlSeeAlso; import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.security.ColumnVisibility; @@ -14,7 +13,6 @@ import io.protostuff.Message; @XmlAccessorType(XmlAccessType.NONE) -@XmlSeeAlso(DefaultField.class) public abstract class FieldBase implements HasMarkings, Message { protected transient Markings markings; diff --git a/web-services/client/src/main/java/datawave/webservice/result/keyword/DefaultTagCloud.java b/web-services/client/src/main/java/datawave/webservice/result/keyword/DefaultTagCloud.java index 9f8df74deae..10561fdc2e5 100644 --- a/web-services/client/src/main/java/datawave/webservice/result/keyword/DefaultTagCloud.java +++ b/web-services/client/src/main/java/datawave/webservice/result/keyword/DefaultTagCloud.java @@ -19,6 +19,7 @@ import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; import datawave.marking.AccessExpressionMarkings; +import datawave.marking.Markings; import datawave.webservice.query.result.event.MapSchema; import datawave.webservice.xml.util.StringMapAdapter; import io.protostuff.Input; @@ -32,6 +33,9 @@ public class DefaultTagCloud extends TagCloudBase markings; + @XmlElement(name = "language") private String language = null; @@ -43,6 +47,16 @@ public class DefaultTagCloud extends TagCloudBase tags = null; + @Override + public Markings getMarkings() { + return markings; + } + + @Override + public void setMarkings(Markings markings) { + this.markings = markings; + } + @Override public String getLanguage() { return language; @@ -129,6 +143,7 @@ public void mergeFrom(Input input, DefaultTagCloud message) throws IOException { while ((number = input.readFieldNumber(this)) != 0) { switch (number) { case 1: + message.markings = AccessExpressionMarkings.builder().build(); input.mergeObject((AccessExpressionMarkings) message.markings, AccessExpressionMarkings.SCHEMA); break; case 2: From 8e5fda595511d3519cb3aa58ec2a7cd4ee367760 Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Tue, 12 May 2026 11:49:13 -0400 Subject: [PATCH 51/63] remove snapshots for unaffected modules --- .../services/accumulo/service/pom.xml | 4 ++-- microservices/services/audit/service/pom.xml | 8 ++++---- .../services/authorization/service/pom.xml | 6 +++--- microservices/services/config/pom.xml | 2 +- .../services/dictionary/service/pom.xml | 2 +- .../services/file-provider/service/pom.xml | 2 +- .../services/hazelcast/client/pom.xml | 2 +- .../services/hazelcast/common/pom.xml | 4 ++-- .../services/hazelcast/service/pom.xml | 2 +- .../mapreduce-query/jobs/core/pom.xml | 4 ++-- .../services/mapreduce-query/service/pom.xml | 2 +- .../services/modification/api/pom.xml | 2 +- .../services/modification/service/pom.xml | 4 ++-- .../services/query-executor/service/pom.xml | 2 +- .../services/query-metric/service/pom.xml | 6 +++--- microservices/services/query/service/pom.xml | 6 +++--- microservices/starters/audit/pom.xml | 4 ++-- microservices/starters/cached-results/pom.xml | 8 ++++---- microservices/starters/datawave/pom.xml | 6 +++--- microservices/starters/metadata/pom.xml | 2 +- microservices/starters/query-metric/pom.xml | 2 +- microservices/starters/query/pom.xml | 10 +++++----- pom.xml | 19 ++++--------------- 23 files changed, 49 insertions(+), 60 deletions(-) diff --git a/microservices/services/accumulo/service/pom.xml b/microservices/services/accumulo/service/pom.xml index 1d72b90e669..ebd73cb83a5 100644 --- a/microservices/services/accumulo/service/pom.xml +++ b/microservices/services/accumulo/service/pom.xml @@ -26,8 +26,8 @@ 5.2.0 7.39.0-SNAPSHOT 4.0.2-SNAPSHOT - 4.0.3-SNAPSHOT - 4.0.3-SNAPSHOT + 4.0.2 + 4.0.2 4.0.8-SNAPSHOT 4.0.6-SNAPSHOT 31.1-jre diff --git a/microservices/services/audit/service/pom.xml b/microservices/services/audit/service/pom.xml index 7e0cfb116a0..445adde22fb 100644 --- a/microservices/services/audit/service/pom.xml +++ b/microservices/services/audit/service/pom.xml @@ -22,10 +22,10 @@ 2.18.0 3.20.0 1.9 - 4.0.3-SNAPSHOT - 4.0.3-SNAPSHOT - 4.0.5-SNAPSHOT - 4.0.5-SNAPSHOT + 4.0.2 + 4.0.2 + 4.0.4 + 4.0.4 4.0.8-SNAPSHOT 2.0.3 31.1-jre diff --git a/microservices/services/authorization/service/pom.xml b/microservices/services/authorization/service/pom.xml index be1c5a25919..681fe1a1a6b 100644 --- a/microservices/services/authorization/service/pom.xml +++ b/microservices/services/authorization/service/pom.xml @@ -21,9 +21,9 @@ datawave.microservice.authorization.AuthorizationService 3.20.0 7.39.0-SNAPSHOT - 4.0.3-SNAPSHOT - 4.0.5-SNAPSHOT - 4.0.5-SNAPSHOT + 4.0.2 + 4.0.4 + 4.0.4 4.0.8-SNAPSHOT 31.1-jre diff --git a/microservices/services/config/pom.xml b/microservices/services/config/pom.xml index 2d35b340d7a..04eb26e230b 100644 --- a/microservices/services/config/pom.xml +++ b/microservices/services/config/pom.xml @@ -25,7 +25,7 @@ datawave.microservice.config.server.ConfigServerApplication - 4.0.8-SNAPSHOT + 4.0.7 5.13.5.202508271544-r 0.2.26 diff --git a/microservices/services/dictionary/service/pom.xml b/microservices/services/dictionary/service/pom.xml index eb52dee03fc..efb4fb2f69b 100644 --- a/microservices/services/dictionary/service/pom.xml +++ b/microservices/services/dictionary/service/pom.xml @@ -29,7 +29,7 @@ 3.20.0 5.2.0 7.39.0-SNAPSHOT - 4.0.3-SNAPSHOT + 4.0.2 4.0.7-SNAPSHOT 4.0.8-SNAPSHOT 3.0.5-SNAPSHOT diff --git a/microservices/services/file-provider/service/pom.xml b/microservices/services/file-provider/service/pom.xml index bdc7325ca40..c639e2d9ce6 100644 --- a/microservices/services/file-provider/service/pom.xml +++ b/microservices/services/file-provider/service/pom.xml @@ -19,7 +19,7 @@ datawave.microservice.fileProvider.FileProviderService - 4.0.8-SNAPSHOT + 4.0.7 2.25.3 diff --git a/microservices/services/hazelcast/client/pom.xml b/microservices/services/hazelcast/client/pom.xml index d4dd47e61e7..e2c5677ce6a 100644 --- a/microservices/services/hazelcast/client/pom.xml +++ b/microservices/services/hazelcast/client/pom.xml @@ -17,7 +17,7 @@ https://github.com/NationalSecurityAgency/datawave - 4.0.5-SNAPSHOT + 4.0.4 2.2.3 diff --git a/microservices/services/hazelcast/common/pom.xml b/microservices/services/hazelcast/common/pom.xml index 155611205d0..0d8905ba4a6 100644 --- a/microservices/services/hazelcast/common/pom.xml +++ b/microservices/services/hazelcast/common/pom.xml @@ -17,8 +17,8 @@ https://github.com/NationalSecurityAgency/datawave - 4.0.8-SNAPSHOT - 4.0.3-SNAPSHOT + 4.0.7 + 4.0.2 2.25.3 diff --git a/microservices/services/hazelcast/service/pom.xml b/microservices/services/hazelcast/service/pom.xml index e161953f3b2..15de2603381 100644 --- a/microservices/services/hazelcast/service/pom.xml +++ b/microservices/services/hazelcast/service/pom.xml @@ -19,7 +19,7 @@ datawave.microservice.cached.server.HazelcastService - 4.0.5-SNAPSHOT + 4.0.4 2.2.3 2.25.3 diff --git a/microservices/services/mapreduce-query/jobs/core/pom.xml b/microservices/services/mapreduce-query/jobs/core/pom.xml index 5e2e144846f..c4c38d98024 100644 --- a/microservices/services/mapreduce-query/jobs/core/pom.xml +++ b/microservices/services/mapreduce-query/jobs/core/pom.xml @@ -18,9 +18,9 @@ https://github.com/NationalSecurityAgency/datawave - 7.39.0-SNAPSHOT + 7.33.1 1.0.0 - 1.0.11-SNAPSHOT + 1.0.10 3.9.2 diff --git a/microservices/services/mapreduce-query/service/pom.xml b/microservices/services/mapreduce-query/service/pom.xml index 6d992d8c81e..2af1396c3ce 100644 --- a/microservices/services/mapreduce-query/service/pom.xml +++ b/microservices/services/mapreduce-query/service/pom.xml @@ -23,7 +23,7 @@ nationalsecurityagency/ datawave.microservice.query.mapreduce.MapReduceQueryService 7.39.0-SNAPSHOT - 4.0.3-SNAPSHOT + 4.0.2 1.0.5-SNAPSHOT 4.0.8-SNAPSHOT 1.0.11-SNAPSHOT diff --git a/microservices/services/modification/api/pom.xml b/microservices/services/modification/api/pom.xml index fb2d35acd0a..ecfb10259ef 100644 --- a/microservices/services/modification/api/pom.xml +++ b/microservices/services/modification/api/pom.xml @@ -18,7 +18,7 @@ 2.1.4-5792fed3-bulkv2 - 7.39.0-SNAPSHOT + 7.33.1 31.1-jre diff --git a/microservices/services/modification/service/pom.xml b/microservices/services/modification/service/pom.xml index 5ae8e7169ce..5614309f0b7 100644 --- a/microservices/services/modification/service/pom.xml +++ b/microservices/services/modification/service/pom.xml @@ -20,8 +20,8 @@ datawave.microservice.modification.ModificationService 7.39.0-SNAPSHOT - 4.0.3-SNAPSHOT - 1.0.2-SNAPSHOT + 4.0.2 + 1.0.1 1.0.2-SNAPSHOT 4.0.8-SNAPSHOT 3.0.5-SNAPSHOT diff --git a/microservices/services/query-executor/service/pom.xml b/microservices/services/query-executor/service/pom.xml index 17e8f2df18f..553e3654005 100644 --- a/microservices/services/query-executor/service/pom.xml +++ b/microservices/services/query-executor/service/pom.xml @@ -24,7 +24,7 @@ 3.20.0 5.2.0 7.39.0-SNAPSHOT - 4.0.3-SNAPSHOT + 4.0.2 1.0.2-SNAPSHOT 1.0.10-SNAPSHOT 4.1.4-SNAPSHOT diff --git a/microservices/services/query-metric/service/pom.xml b/microservices/services/query-metric/service/pom.xml index 96b837371f2..2a9f02f5a4c 100644 --- a/microservices/services/query-metric/service/pom.xml +++ b/microservices/services/query-metric/service/pom.xml @@ -26,9 +26,9 @@ 3.20.0 7.39.0-SNAPSHOT 4.0.2-SNAPSHOT - 4.0.3-SNAPSHOT - 4.0.3-SNAPSHOT - 4.0.5-SNAPSHOT + 4.0.2 + 4.0.2 + 4.0.4 1.0.2-SNAPSHOT 4.1.4-SNAPSHOT 4.0.8-SNAPSHOT diff --git a/microservices/services/query/service/pom.xml b/microservices/services/query/service/pom.xml index 9ad2a15fac7..08c27501b04 100644 --- a/microservices/services/query/service/pom.xml +++ b/microservices/services/query/service/pom.xml @@ -25,9 +25,9 @@ 2.18.0 3.20.0 7.39.0-SNAPSHOT - 4.0.3-SNAPSHOT - 4.0.3-SNAPSHOT - 4.0.5-SNAPSHOT + 4.0.2 + 4.0.2 + 4.0.4 1.0.2-SNAPSHOT 4.1.4-SNAPSHOT 4.0.8-SNAPSHOT diff --git a/microservices/starters/audit/pom.xml b/microservices/starters/audit/pom.xml index ec8c6b89548..0cbc90af787 100644 --- a/microservices/starters/audit/pom.xml +++ b/microservices/starters/audit/pom.xml @@ -26,8 +26,8 @@ 1.9.25.1 3.20.0 7.39.0-SNAPSHOT - 4.0.3-SNAPSHOT - 4.0.3-SNAPSHOT + 4.0.2 + 4.0.2 4.0.8-SNAPSHOT 31.1-jre 2.2 diff --git a/microservices/starters/cached-results/pom.xml b/microservices/starters/cached-results/pom.xml index 3cb0810ac77..e5a4b5b51cb 100644 --- a/microservices/starters/cached-results/pom.xml +++ b/microservices/starters/cached-results/pom.xml @@ -26,10 +26,10 @@ 4.3 3.20.0 7.39.0-SNAPSHOT - 4.0.3-SNAPSHOT - 4.0.3-SNAPSHOT - 4.0.5-SNAPSHOT - 4.0.5-SNAPSHOT + 4.0.2 + 4.0.2 + 4.0.4 + 4.0.4 1.0.2-SNAPSHOT 4.0.8-SNAPSHOT 4.0.6-SNAPSHOT diff --git a/microservices/starters/datawave/pom.xml b/microservices/starters/datawave/pom.xml index 03d2c75677a..bc05bb9718e 100644 --- a/microservices/starters/datawave/pom.xml +++ b/microservices/starters/datawave/pom.xml @@ -27,9 +27,9 @@ 1.9.25.1 1.9.4 7.39.0-SNAPSHOT - 4.0.3-SNAPSHOT - 3.0.1-SNAPSHOT - 4.0.3-SNAPSHOT + 4.0.2 + 3.0.0 + 4.0.2 31.1-jre 2.2 4.0.4 diff --git a/microservices/starters/metadata/pom.xml b/microservices/starters/metadata/pom.xml index a621f5fed05..ac3a09e9de3 100644 --- a/microservices/starters/metadata/pom.xml +++ b/microservices/starters/metadata/pom.xml @@ -26,7 +26,7 @@ 2.1.4-5792fed3-bulkv2 7.39.0-SNAPSHOT 4.0.8-SNAPSHOT - 4.0.3-SNAPSHOT + 4.0.2 2.2.42 diff --git a/microservices/starters/query-metric/pom.xml b/microservices/starters/query-metric/pom.xml index ca61ffe3afa..256125f3240 100644 --- a/microservices/starters/query-metric/pom.xml +++ b/microservices/starters/query-metric/pom.xml @@ -25,7 +25,7 @@ 3.20.0 7.39.0-SNAPSHOT - 4.0.3-SNAPSHOT + 4.0.2 4.1.4-SNAPSHOT 4.0.8-SNAPSHOT 31.1-jre diff --git a/microservices/starters/query/pom.xml b/microservices/starters/query/pom.xml index 8037ad78d46..59463e312c6 100644 --- a/microservices/starters/query/pom.xml +++ b/microservices/starters/query/pom.xml @@ -27,12 +27,12 @@ 2.1.4-5792fed3-bulkv2 3.20.0 7.39.0-SNAPSHOT - 4.0.3-SNAPSHOT - 4.0.3-SNAPSHOT + 4.0.2 + 4.0.2 4.0.7-SNAPSHOT - 4.0.5-SNAPSHOT - 4.0.5-SNAPSHOT - 1.0.2-SNAPSHOT + 4.0.4 + 4.0.4 + 1.0.1 1.0.2-SNAPSHOT 4.1.4-SNAPSHOT 4.0.8-SNAPSHOT diff --git a/pom.xml b/pom.xml index ed1064f965d..8a4b19fa174 100644 --- a/pom.xml +++ b/pom.xml @@ -76,24 +76,13 @@ 5.9.0 5.9.0 4.0.2-SNAPSHOT - 4.0.3-SNAPSHOT - 4.0.3-SNAPSHOT + 4.0.2 + 4.0.2 4.0.7-SNAPSHOT - 4.0.5-SNAPSHOT - 4.0.5-SNAPSHOT - 1.0.2-SNAPSHOT - 3.0.1-SNAPSHOT - 1.0.2-SNAPSHOT + 1.0.1 + 3.0.0 1.0.2-SNAPSHOT - 1.0.10-SNAPSHOT 4.1.4-SNAPSHOT - 4.0.8-SNAPSHOT - 4.0.6-SNAPSHOT - 4.0.3-SNAPSHOT - 1.0.13-SNAPSHOT - 3.0.5-SNAPSHOT - 1.0.11-SNAPSHOT - 3.0.6-SNAPSHOT 1.9.0 3.6.3 5.2.0 From c7ca57a9b4797b0d256e068b31bd4ed57eac14fd Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Tue, 12 May 2026 13:41:59 -0400 Subject: [PATCH 52/63] remove extraneous test change --- .../java/datawave/query/tables/RemoteEventQueryLogicIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/RemoteEventQueryLogicIT.java b/warehouse/query-core/src/test/java/datawave/query/tables/RemoteEventQueryLogicIT.java index d35aac7bae8..93d046c9eb0 100644 --- a/warehouse/query-core/src/test/java/datawave/query/tables/RemoteEventQueryLogicIT.java +++ b/warehouse/query-core/src/test/java/datawave/query/tables/RemoteEventQueryLogicIT.java @@ -161,7 +161,7 @@ public void testSocketTimeout() { assertTrue(r.getException() instanceof RemoteTimeoutQueryException); } - @Test(timeout = 35000) + @Test(timeout = 30000) public void testDefaultConnectionPoolTimeout() { RemoteHttpService remoteHttpService = (RemoteHttpService) logic.getRemoteQueryService(); RemoteHttpServiceConfiguration remoteConfig = remoteHttpService.getConfig(); From 1158ca1d30fea089ab06243af7ed27b9bbd2407e Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Tue, 12 May 2026 15:06:58 -0400 Subject: [PATCH 53/63] use -JFA-20260512-1 modifier in POMs --- common-test/pom.xml | 2 +- contrib/datawave-quickstart/docker/pom.xml | 2 +- core/annotation/pom.xml | 2 +- core/base-rest-responses/pom.xml | 2 +- core/cached-results/pom.xml | 2 +- core/common-util/pom.xml | 2 +- core/common/pom.xml | 2 +- core/connection-pool/pom.xml | 2 +- core/map-reduce/pom.xml | 2 +- core/modification/pom.xml | 2 +- core/pom.xml | 2 +- core/query/pom.xml | 2 +- core/utils/accumulo-utils/pom.xml | 2 +- core/utils/common-utils/pom.xml | 2 +- core/utils/metadata-utils/pom.xml | 2 +- core/utils/pom.xml | 2 +- core/utils/type-utils/pom.xml | 2 +- coverage/pom.xml | 2 +- docs/pom.xml | 2 +- microservices/pom.xml | 2 +- microservices/services/accumulo/api/pom.xml | 4 ++-- .../services/accumulo/service/pom.xml | 10 +++++----- microservices/services/audit/api/pom.xml | 2 +- microservices/services/audit/service/pom.xml | 4 ++-- .../services/authorization/service/pom.xml | 6 +++--- microservices/services/dictionary/api/pom.xml | 4 ++-- .../services/dictionary/service/pom.xml | 10 +++++----- .../services/mapreduce-query/jobs/core/pom.xml | 2 +- .../services/mapreduce-query/service/pom.xml | 10 +++++----- .../services/modification/service/pom.xml | 10 +++++----- microservices/services/pom.xml | 2 +- .../services/query-executor/api/pom.xml | 6 +++--- .../services/query-executor/service/pom.xml | 16 ++++++++-------- .../services/query-metric/api/pom.xml | 6 +++--- .../services/query-metric/service/pom.xml | 16 ++++++++-------- microservices/services/query/api/pom.xml | 4 ++-- microservices/services/query/service/pom.xml | 18 +++++++++--------- microservices/starters/audit/pom.xml | 6 +++--- microservices/starters/cached-results/pom.xml | 12 ++++++------ microservices/starters/datawave/pom.xml | 4 ++-- microservices/starters/metadata/pom.xml | 6 +++--- microservices/starters/pom.xml | 2 +- microservices/starters/query-metric/pom.xml | 8 ++++---- microservices/starters/query/pom.xml | 16 ++++++++-------- pom.xml | 10 +++++----- warehouse/accumulo-extensions/pom.xml | 2 +- warehouse/age-off-utils/pom.xml | 2 +- warehouse/age-off/pom.xml | 2 +- warehouse/assemble/datawave/pom.xml | 2 +- warehouse/assemble/pom.xml | 2 +- warehouse/assemble/webservice/pom.xml | 2 +- warehouse/common/pom.xml | 2 +- warehouse/core/pom.xml | 2 +- warehouse/data-dictionary-core/pom.xml | 2 +- warehouse/edge-dictionary-core/pom.xml | 2 +- .../edge-model-configuration-core/pom.xml | 2 +- warehouse/index-stats/pom.xml | 2 +- warehouse/ingest-annotation/pom.xml | 2 +- warehouse/ingest-configuration/pom.xml | 2 +- warehouse/ingest-core/pom.xml | 2 +- warehouse/ingest-csv/pom.xml | 2 +- warehouse/ingest-json/pom.xml | 2 +- warehouse/ingest-nyctlc/pom.xml | 2 +- warehouse/ingest-scripts/pom.xml | 2 +- warehouse/ingest-ssdeep/pom.xml | 2 +- warehouse/ingest-wikipedia/pom.xml | 2 +- warehouse/keyword-common/pom.xml | 2 +- warehouse/metrics-core/pom.xml | 2 +- warehouse/ops-tools/config-compare/pom.xml | 2 +- warehouse/ops-tools/index-validation/pom.xml | 2 +- warehouse/ops-tools/pom.xml | 2 +- warehouse/pom.xml | 2 +- warehouse/query-core/pom.xml | 2 +- warehouse/regression-testing/pom.xml | 2 +- warehouse/ssdeep-common/pom.xml | 2 +- web-services/accumulo/pom.xml | 2 +- web-services/annotations/pom.xml | 2 +- web-services/atom/pom.xml | 2 +- web-services/cached-results/pom.xml | 2 +- web-services/client/pom.xml | 2 +- web-services/common-util/pom.xml | 2 +- web-services/common/pom.xml | 2 +- web-services/deploy/application/pom.xml | 2 +- web-services/deploy/configuration/pom.xml | 2 +- web-services/deploy/docs/pom.xml | 2 +- web-services/deploy/pom.xml | 2 +- .../spring-framework-integration/pom.xml | 2 +- web-services/dictionary/pom.xml | 2 +- web-services/examples/client-login/pom.xml | 2 +- web-services/examples/http-client/pom.xml | 2 +- web-services/examples/jms-client/pom.xml | 2 +- web-services/examples/pom.xml | 2 +- web-services/examples/query-war/pom.xml | 2 +- web-services/metrics/pom.xml | 2 +- web-services/model/pom.xml | 2 +- web-services/modification/pom.xml | 2 +- web-services/pom.xml | 2 +- web-services/query-websocket/pom.xml | 2 +- web-services/query/pom.xml | 2 +- web-services/rest-api/pom.xml | 2 +- web-services/security/pom.xml | 2 +- web-services/web-root/pom.xml | 2 +- 102 files changed, 174 insertions(+), 174 deletions(-) diff --git a/common-test/pom.xml b/common-test/pom.xml index 696661d047e..897be3e62eb 100644 --- a/common-test/pom.xml +++ b/common-test/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-common-test ${project.artifactId} diff --git a/contrib/datawave-quickstart/docker/pom.xml b/contrib/datawave-quickstart/docker/pom.xml index 7bda04c19b0..b1f13359f42 100644 --- a/contrib/datawave-quickstart/docker/pom.xml +++ b/contrib/datawave-quickstart/docker/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 ../../../pom.xml quickstart diff --git a/core/annotation/pom.xml b/core/annotation/pom.xml index 2cf18c8c5c6..18d753eaffd 100644 --- a/core/annotation/pom.xml +++ b/core/annotation/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-core-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-core-annotation diff --git a/core/base-rest-responses/pom.xml b/core/base-rest-responses/pom.xml index e5b1186af92..0d4a694d0ea 100644 --- a/core/base-rest-responses/pom.xml +++ b/core/base-rest-responses/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-core-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 base-rest-responses ${project.artifactId} diff --git a/core/cached-results/pom.xml b/core/cached-results/pom.xml index c9e7a44f4f3..c366aebbf4b 100644 --- a/core/cached-results/pom.xml +++ b/core/cached-results/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-core-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-core-cached-results ${project.artifactId} diff --git a/core/common-util/pom.xml b/core/common-util/pom.xml index 5d815e50965..2a45b5960f0 100644 --- a/core/common-util/pom.xml +++ b/core/common-util/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-core-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-core-common-util ${project.artifactId} diff --git a/core/common/pom.xml b/core/common/pom.xml index 4cc11343d62..ae688a5b9c8 100644 --- a/core/common/pom.xml +++ b/core/common/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-core-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-core-common ${project.artifactId} diff --git a/core/connection-pool/pom.xml b/core/connection-pool/pom.xml index 3eba6dba432..b4a878f5868 100644 --- a/core/connection-pool/pom.xml +++ b/core/connection-pool/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-core-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-core-connection-pool ${project.artifactId} diff --git a/core/map-reduce/pom.xml b/core/map-reduce/pom.xml index 0f6d2dc70bb..540e3fbe736 100644 --- a/core/map-reduce/pom.xml +++ b/core/map-reduce/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-core-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-core-map-reduce ${project.artifactId} diff --git a/core/modification/pom.xml b/core/modification/pom.xml index 820b45fa3ec..8c65a37e854 100644 --- a/core/modification/pom.xml +++ b/core/modification/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-core-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-core-modification ${project.artifactId} diff --git a/core/pom.xml b/core/pom.xml index a2a2931c052..2f1e4efc57d 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 gov.nsa.datawave.core datawave-core-parent diff --git a/core/query/pom.xml b/core/query/pom.xml index 9ece9a86c8e..c27e57b4308 100644 --- a/core/query/pom.xml +++ b/core/query/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-core-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-core-query ${project.artifactId} diff --git a/core/utils/accumulo-utils/pom.xml b/core/utils/accumulo-utils/pom.xml index 0d015ab8147..f0db83c9df3 100644 --- a/core/utils/accumulo-utils/pom.xml +++ b/core/utils/accumulo-utils/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-utils-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 accumulo-utils ${project.artifactId} diff --git a/core/utils/common-utils/pom.xml b/core/utils/common-utils/pom.xml index 97e5de2c0ea..a29bf4777cc 100644 --- a/core/utils/common-utils/pom.xml +++ b/core/utils/common-utils/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-utils-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 common-utils ${project.artifactId} diff --git a/core/utils/metadata-utils/pom.xml b/core/utils/metadata-utils/pom.xml index 944de13daf9..d679608a938 100644 --- a/core/utils/metadata-utils/pom.xml +++ b/core/utils/metadata-utils/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-utils-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 metadata-utils ${project.artifactId} diff --git a/core/utils/pom.xml b/core/utils/pom.xml index 4c06579942d..107e7127e7a 100644 --- a/core/utils/pom.xml +++ b/core/utils/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-core-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 gov.nsa.datawave.core datawave-utils-parent diff --git a/core/utils/type-utils/pom.xml b/core/utils/type-utils/pom.xml index c3fe8995d22..dadfca6994e 100644 --- a/core/utils/type-utils/pom.xml +++ b/core/utils/type-utils/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-utils-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 type-utils ${project.artifactId} diff --git a/coverage/pom.xml b/coverage/pom.xml index 8c691e6d7c9..4991bb74183 100644 --- a/coverage/pom.xml +++ b/coverage/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 coverage Datawave Code Coverage Aggregate Report diff --git a/docs/pom.xml b/docs/pom.xml index 96c24783ea5..70e046c30b1 100644 --- a/docs/pom.xml +++ b/docs/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-docs diff --git a/microservices/pom.xml b/microservices/pom.xml index c39adb3edb8..efc9c0d4ac1 100644 --- a/microservices/pom.xml +++ b/microservices/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 gov.nsa.datawave.microservice datawave-microservice-build-parent diff --git a/microservices/services/accumulo/api/pom.xml b/microservices/services/accumulo/api/pom.xml index 23d1c77ad2b..295a9dccff7 100644 --- a/microservices/services/accumulo/api/pom.xml +++ b/microservices/services/accumulo/api/pom.xml @@ -8,7 +8,7 @@ ../../../microservice-parent/pom.xml accumulo-api - 4.0.2-SNAPSHOT + 4.0.2-JFA-20260512-1 https://code.nsa.gov/datawave-accumulo-service scm:git:${project.scm.url} @@ -19,7 +19,7 @@ http://webservice.datawave.nsa/v1 1.17.1 - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 2.14.3 2.3.3 diff --git a/microservices/services/accumulo/service/pom.xml b/microservices/services/accumulo/service/pom.xml index ebd73cb83a5..a41e5d1ec7e 100644 --- a/microservices/services/accumulo/service/pom.xml +++ b/microservices/services/accumulo/service/pom.xml @@ -8,7 +8,7 @@ ../../../microservice-service-parent/pom.xml accumulo-service - 4.0.5-SNAPSHOT + 4.0.5-JFA-20260512-1 DATAWAVE Accumulo Microservice https://code.nsa.gov/datawave-accumulo-service @@ -24,12 +24,12 @@ 3.20.0 5.2.0 5.2.0 - 7.39.0-SNAPSHOT - 4.0.2-SNAPSHOT + 7.39.0-JFA-20260512-1 + 4.0.2-JFA-20260512-1 4.0.2 4.0.2 - 4.0.8-SNAPSHOT - 4.0.6-SNAPSHOT + 4.0.8-JFA-20260512-1 + 4.0.6-JFA-20260512-1 31.1-jre 3.3.4 4.0.4 diff --git a/microservices/services/audit/api/pom.xml b/microservices/services/audit/api/pom.xml index fa8058ce07a..63b4c953206 100644 --- a/microservices/services/audit/api/pom.xml +++ b/microservices/services/audit/api/pom.xml @@ -8,7 +8,7 @@ ../../../microservice-parent/pom.xml audit-api - 4.0.3-SNAPSHOT + 4.0.3-JFA-20260512-1 https://code.nsa.gov/datawave-audit-service scm:git:${project.scm.url} diff --git a/microservices/services/audit/service/pom.xml b/microservices/services/audit/service/pom.xml index 445adde22fb..e951c67a9e2 100644 --- a/microservices/services/audit/service/pom.xml +++ b/microservices/services/audit/service/pom.xml @@ -8,7 +8,7 @@ ../../../microservice-service-parent/pom.xml audit-service - 4.0.5-SNAPSHOT + 4.0.5-JFA-20260512-1 DATAWAVE Auditing Microservice https://code.nsa.gov/datawave-audit-service @@ -26,7 +26,7 @@ 4.0.2 4.0.4 4.0.4 - 4.0.8-SNAPSHOT + 4.0.8-JFA-20260512-1 2.0.3 31.1-jre 3.3.4 diff --git a/microservices/services/authorization/service/pom.xml b/microservices/services/authorization/service/pom.xml index 681fe1a1a6b..bdb479c1e10 100644 --- a/microservices/services/authorization/service/pom.xml +++ b/microservices/services/authorization/service/pom.xml @@ -8,7 +8,7 @@ ../../../microservice-service-parent/pom.xml authorization-service - 4.0.7-SNAPSHOT + 4.0.7-JFA-20260512-1 DATAWAVE Authorization Microservice https://code.nsa.gov/datawave-authorization-service @@ -20,11 +20,11 @@ datawave.microservice.authorization.AuthorizationService 3.20.0 - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 4.0.2 4.0.4 4.0.4 - 4.0.8-SNAPSHOT + 4.0.8-JFA-20260512-1 31.1-jre diff --git a/microservices/services/dictionary/api/pom.xml b/microservices/services/dictionary/api/pom.xml index fa385859928..7ec8a7bf900 100644 --- a/microservices/services/dictionary/api/pom.xml +++ b/microservices/services/dictionary/api/pom.xml @@ -8,7 +8,7 @@ ../../../microservice-parent/pom.xml dictionary-api - 4.0.7-SNAPSHOT + 4.0.7-JFA-20260512-1 https://code.nsa.gov/datawave-dictionary-service scm:git:${project.scm.url} @@ -19,7 +19,7 @@ http://webservice.datawave.nsa/v1 3.20.0 - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 31.1-jre 2.3.3 1.6.2 diff --git a/microservices/services/dictionary/service/pom.xml b/microservices/services/dictionary/service/pom.xml index efb4fb2f69b..bd175f652f1 100644 --- a/microservices/services/dictionary/service/pom.xml +++ b/microservices/services/dictionary/service/pom.xml @@ -8,7 +8,7 @@ ../../../microservice-service-parent/pom.xml dictionary-service - 4.0.12-SNAPSHOT + 4.0.12-JFA-20260512-1 DATAWAVE Dictionary Service https://code.nsa.gov/datawave-dictionary-service @@ -28,11 +28,11 @@ 3.2.2 3.20.0 5.2.0 - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 4.0.2 - 4.0.7-SNAPSHOT - 4.0.8-SNAPSHOT - 3.0.5-SNAPSHOT + 4.0.7-JFA-20260512-1 + 4.0.8-JFA-20260512-1 + 3.0.5-JFA-20260512-1 31.1-jre 3.3.4 4.0.4 diff --git a/microservices/services/mapreduce-query/jobs/core/pom.xml b/microservices/services/mapreduce-query/jobs/core/pom.xml index c4c38d98024..12c644674ff 100644 --- a/microservices/services/mapreduce-query/jobs/core/pom.xml +++ b/microservices/services/mapreduce-query/jobs/core/pom.xml @@ -8,7 +8,7 @@ ../../../../microservice-service-parent/pom.xml mapreduce-query-core-job - 1.0.5-SNAPSHOT + 1.0.5-JFA-20260512-1 DATAWAVE MapReduce Query Core Job https://github.com/NationalSecurityAgency/datawave-mapreduce-query-service diff --git a/microservices/services/mapreduce-query/service/pom.xml b/microservices/services/mapreduce-query/service/pom.xml index 2af1396c3ce..1e087a1108d 100644 --- a/microservices/services/mapreduce-query/service/pom.xml +++ b/microservices/services/mapreduce-query/service/pom.xml @@ -8,7 +8,7 @@ ../../../microservice-service-parent/pom.xml mapreduce-query-service - 1.0.5-SNAPSHOT + 1.0.5-JFA-20260512-1 DATAWAVE MapReduce Query Microservice https://github.com/NationalSecurityAgency/datawave-mapreduce-query-service @@ -22,11 +22,11 @@ datawave/ nationalsecurityagency/ datawave.microservice.query.mapreduce.MapReduceQueryService - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 4.0.2 - 1.0.5-SNAPSHOT - 4.0.8-SNAPSHOT - 1.0.11-SNAPSHOT + 1.0.5-JFA-20260512-1 + 4.0.8-JFA-20260512-1 + 1.0.11-JFA-20260512-1 3.3.4 2.25.3 diff --git a/microservices/services/modification/service/pom.xml b/microservices/services/modification/service/pom.xml index 5614309f0b7..f3875f54805 100644 --- a/microservices/services/modification/service/pom.xml +++ b/microservices/services/modification/service/pom.xml @@ -8,7 +8,7 @@ ../../../microservice-service-parent/pom.xml modification-service - 1.0.5-SNAPSHOT + 1.0.5-JFA-20260512-1 DATAWAVE Modification Service https://code.nsa.gov/datawave-modification-service @@ -19,12 +19,12 @@ datawave.microservice.modification.ModificationService - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 4.0.2 1.0.1 - 1.0.2-SNAPSHOT - 4.0.8-SNAPSHOT - 3.0.5-SNAPSHOT + 1.0.2-JFA-20260512-1 + 4.0.8-JFA-20260512-1 + 3.0.5-JFA-20260512-1 4.0.4 2.25.3 diff --git a/microservices/services/pom.xml b/microservices/services/pom.xml index e997bae4481..3a88185a744 100644 --- a/microservices/services/pom.xml +++ b/microservices/services/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-build-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-microservice-service-build-parent pom diff --git a/microservices/services/query-executor/api/pom.xml b/microservices/services/query-executor/api/pom.xml index 235ed3a37c3..57031a57eca 100644 --- a/microservices/services/query-executor/api/pom.xml +++ b/microservices/services/query-executor/api/pom.xml @@ -8,7 +8,7 @@ ../../../microservice-parent/pom.xml query-executor-api - 1.0.10-SNAPSHOT + 1.0.10-JFA-20260512-1 https://github.com/NationalSecurityAgency/datawave-query-executor-service scm:git:${project.scm.url} @@ -19,8 +19,8 @@ http://webservice.datawave/v1 3.20.0 - 7.39.0-SNAPSHOT - 1.0.11-SNAPSHOT + 7.39.0-JFA-20260512-1 + 1.0.11-JFA-20260512-1 2.3.3 diff --git a/microservices/services/query-executor/service/pom.xml b/microservices/services/query-executor/service/pom.xml index 553e3654005..de79397520a 100644 --- a/microservices/services/query-executor/service/pom.xml +++ b/microservices/services/query-executor/service/pom.xml @@ -8,7 +8,7 @@ ../../../microservice-service-parent/pom.xml query-executor-service - 1.0.12-SNAPSHOT + 1.0.12-JFA-20260512-1 DATAWAVE Query Executor Service https://github.com/NationalSecurityAgency/datawave-query-executor-service @@ -23,14 +23,14 @@ 4.3 3.20.0 5.2.0 - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 4.0.2 - 1.0.2-SNAPSHOT - 1.0.10-SNAPSHOT - 4.1.4-SNAPSHOT - 4.0.8-SNAPSHOT - 1.0.11-SNAPSHOT - 3.0.6-SNAPSHOT + 1.0.2-JFA-20260512-1 + 1.0.10-JFA-20260512-1 + 4.1.4-JFA-20260512-1 + 4.0.8-JFA-20260512-1 + 1.0.11-JFA-20260512-1 + 3.0.6-JFA-20260512-1 31.1-jre 3.3.4 4.0.4 diff --git a/microservices/services/query-metric/api/pom.xml b/microservices/services/query-metric/api/pom.xml index 19da3c53d5c..44956db1925 100644 --- a/microservices/services/query-metric/api/pom.xml +++ b/microservices/services/query-metric/api/pom.xml @@ -8,7 +8,7 @@ ../../../microservice-parent/pom.xml query-metric-api - 4.1.4-SNAPSHOT + 4.1.4-JFA-20260512-1 https://code.nsa.gov/datawave-query-metric-service scm:git:${project.scm.url} @@ -21,8 +21,8 @@ ${basedir}/spotbugs-exclude.xml 3.20.0 1.9 - 7.39.0-SNAPSHOT - 1.0.2-SNAPSHOT + 7.39.0-JFA-20260512-1 + 1.0.2-JFA-20260512-1 2.3.6 31.1-jre diff --git a/microservices/services/query-metric/service/pom.xml b/microservices/services/query-metric/service/pom.xml index 2a9f02f5a4c..9fd498006aa 100644 --- a/microservices/services/query-metric/service/pom.xml +++ b/microservices/services/query-metric/service/pom.xml @@ -8,7 +8,7 @@ ../../../microservice-service-parent/pom.xml query-metric-service - 4.1.11-SNAPSHOT + 4.1.11-JFA-20260512-1 DATAWAVE Query Metric Microservice https://code.nsa.gov/datawave-query-metric-service @@ -24,16 +24,16 @@ 1.1.1 3.3 3.20.0 - 7.39.0-SNAPSHOT - 4.0.2-SNAPSHOT + 7.39.0-JFA-20260512-1 + 4.0.2-JFA-20260512-1 4.0.2 4.0.2 4.0.4 - 1.0.2-SNAPSHOT - 4.1.4-SNAPSHOT - 4.0.8-SNAPSHOT - 3.0.5-SNAPSHOT - 3.0.6-SNAPSHOT + 1.0.2-JFA-20260512-1 + 4.1.4-JFA-20260512-1 + 4.0.8-JFA-20260512-1 + 3.0.5-JFA-20260512-1 + 3.0.6-JFA-20260512-1 31.1-jre 3.3.4 2.2.3 diff --git a/microservices/services/query/api/pom.xml b/microservices/services/query/api/pom.xml index 652a277eee0..b0a43187ed0 100644 --- a/microservices/services/query/api/pom.xml +++ b/microservices/services/query/api/pom.xml @@ -8,7 +8,7 @@ ../../../microservice-parent/pom.xml query-api - 1.0.2-SNAPSHOT + 1.0.2-JFA-20260512-1 https://github.com/NationalSecurityAgency/datawave-query-service scm:git:${project.scm.url} @@ -20,7 +20,7 @@ http://webservice.datawave.nsa/v1 4.3 3.20.0 - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 31.1-jre 2.14.3 diff --git a/microservices/services/query/service/pom.xml b/microservices/services/query/service/pom.xml index 08c27501b04..4aa4635d57c 100644 --- a/microservices/services/query/service/pom.xml +++ b/microservices/services/query/service/pom.xml @@ -8,7 +8,7 @@ ../../../microservice-service-parent/pom.xml query-service - 1.0.14-SNAPSHOT + 1.0.14-JFA-20260512-1 DATAWAVE Query Microservice https://github.com/NationalSecurityAgency/datawave-query-service @@ -24,17 +24,17 @@ 1.26.0 2.18.0 3.20.0 - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 4.0.2 4.0.2 4.0.4 - 1.0.2-SNAPSHOT - 4.1.4-SNAPSHOT - 4.0.8-SNAPSHOT - 4.0.6-SNAPSHOT - 1.0.13-SNAPSHOT - 1.0.11-SNAPSHOT - 3.0.6-SNAPSHOT + 1.0.2-JFA-20260512-1 + 4.1.4-JFA-20260512-1 + 4.0.8-JFA-20260512-1 + 4.0.6-JFA-20260512-1 + 1.0.13-JFA-20260512-1 + 1.0.11-JFA-20260512-1 + 3.0.6-JFA-20260512-1 31.1-jre 3.3.4 diff --git a/microservices/starters/audit/pom.xml b/microservices/starters/audit/pom.xml index 0cbc90af787..14a6e1d8f34 100644 --- a/microservices/starters/audit/pom.xml +++ b/microservices/starters/audit/pom.xml @@ -8,7 +8,7 @@ ../../microservice-service-parent/pom.xml spring-boot-starter-datawave-audit - 4.0.6-SNAPSHOT + 4.0.6-JFA-20260512-1 https://code.nsa.gov/datawave-spring-boot-starter-audit @@ -25,10 +25,10 @@ 1.9.25.1 3.20.0 - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 4.0.2 4.0.2 - 4.0.8-SNAPSHOT + 4.0.8-JFA-20260512-1 31.1-jre 2.2 diff --git a/microservices/starters/cached-results/pom.xml b/microservices/starters/cached-results/pom.xml index e5a4b5b51cb..8e1d56f8ad2 100644 --- a/microservices/starters/cached-results/pom.xml +++ b/microservices/starters/cached-results/pom.xml @@ -8,7 +8,7 @@ ../../microservice-service-parent/pom.xml spring-boot-starter-datawave-cached-results - 1.0.13-SNAPSHOT + 1.0.13-JFA-20260512-1 https://code.nsa.gov/datawave-spring-boot-starter-cached-results @@ -25,15 +25,15 @@ 4.3 3.20.0 - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 4.0.2 4.0.2 4.0.4 4.0.4 - 1.0.2-SNAPSHOT - 4.0.8-SNAPSHOT - 4.0.6-SNAPSHOT - 1.0.11-SNAPSHOT + 1.0.2-JFA-20260512-1 + 4.0.8-JFA-20260512-1 + 4.0.6-JFA-20260512-1 + 1.0.11-JFA-20260512-1 9.3.0 diff --git a/microservices/starters/datawave/pom.xml b/microservices/starters/datawave/pom.xml index bc05bb9718e..d79a84334e8 100644 --- a/microservices/starters/datawave/pom.xml +++ b/microservices/starters/datawave/pom.xml @@ -8,7 +8,7 @@ ../../microservice-service-parent/pom.xml spring-boot-starter-datawave - 4.0.8-SNAPSHOT + 4.0.8-JFA-20260512-1 https://code.nsa.gov/datawave-spring-boot-starter @@ -26,7 +26,7 @@ 2.1.4-5792fed3-bulkv2 1.9.25.1 1.9.4 - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 4.0.2 3.0.0 4.0.2 diff --git a/microservices/starters/metadata/pom.xml b/microservices/starters/metadata/pom.xml index ac3a09e9de3..93d41d30ef1 100644 --- a/microservices/starters/metadata/pom.xml +++ b/microservices/starters/metadata/pom.xml @@ -8,7 +8,7 @@ ../../microservice-service-parent/pom.xml spring-boot-starter-datawave-metadata - 3.0.5-SNAPSHOT + 3.0.5-JFA-20260512-1 https://github.com/NationalSecurityAgency/datawave-spring-boot-starter-metadata @@ -24,8 +24,8 @@ 2.1.4-5792fed3-bulkv2 - 7.39.0-SNAPSHOT - 4.0.8-SNAPSHOT + 7.39.0-JFA-20260512-1 + 4.0.8-JFA-20260512-1 4.0.2 2.2.42 diff --git a/microservices/starters/pom.xml b/microservices/starters/pom.xml index 17aa5ff6c64..bac79ca2014 100644 --- a/microservices/starters/pom.xml +++ b/microservices/starters/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-build-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-microservice-starter-build-parent pom diff --git a/microservices/starters/query-metric/pom.xml b/microservices/starters/query-metric/pom.xml index 256125f3240..64ef71b7012 100644 --- a/microservices/starters/query-metric/pom.xml +++ b/microservices/starters/query-metric/pom.xml @@ -8,7 +8,7 @@ ../../microservice-service-parent/pom.xml spring-boot-starter-datawave-query-metric - 3.0.6-SNAPSHOT + 3.0.6-JFA-20260512-1 jar @@ -24,10 +24,10 @@ 3.20.0 - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 4.0.2 - 4.1.4-SNAPSHOT - 4.0.8-SNAPSHOT + 4.1.4-JFA-20260512-1 + 4.0.8-JFA-20260512-1 31.1-jre diff --git a/microservices/starters/query/pom.xml b/microservices/starters/query/pom.xml index 59463e312c6..7a6fa068337 100644 --- a/microservices/starters/query/pom.xml +++ b/microservices/starters/query/pom.xml @@ -8,7 +8,7 @@ ../../microservice-service-parent/pom.xml spring-boot-starter-datawave-query - 1.0.11-SNAPSHOT + 1.0.11-JFA-20260512-1 https://github.com/NationalSecurityAgency/datawave-spring-boot-starter-query @@ -26,18 +26,18 @@ http://webservice.datawave/v1 2.1.4-5792fed3-bulkv2 3.20.0 - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 4.0.2 4.0.2 - 4.0.7-SNAPSHOT + 4.0.7-JFA-20260512-1 4.0.4 4.0.4 1.0.1 - 1.0.2-SNAPSHOT - 4.1.4-SNAPSHOT - 4.0.8-SNAPSHOT - 3.0.5-SNAPSHOT - 3.0.6-SNAPSHOT + 1.0.2-JFA-20260512-1 + 4.1.4-JFA-20260512-1 + 4.0.8-JFA-20260512-1 + 3.0.5-JFA-20260512-1 + 3.0.6-JFA-20260512-1 31.1-jre 3.3.4 diff --git a/pom.xml b/pom.xml index 8a4b19fa174..28c4ec4975f 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 gov.nsa.datawave datawave-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 pom DataWave DataWave is a Java-based ingest and query framework that leverages Apache Accumulo to provide fast, secure access to your data. @@ -75,14 +75,14 @@ 1.11.0 5.9.0 5.9.0 - 4.0.2-SNAPSHOT + 4.0.2-JFA-20260512-1 4.0.2 4.0.2 - 4.0.7-SNAPSHOT + 4.0.7-JFA-20260512-1 1.0.1 3.0.0 - 1.0.2-SNAPSHOT - 4.1.4-SNAPSHOT + 1.0.2-JFA-20260512-1 + 4.1.4-JFA-20260512-1 1.9.0 3.6.3 5.2.0 diff --git a/warehouse/accumulo-extensions/pom.xml b/warehouse/accumulo-extensions/pom.xml index 3471090617d..a8570ed0bb8 100644 --- a/warehouse/accumulo-extensions/pom.xml +++ b/warehouse/accumulo-extensions/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-accumulo-extensions ${project.artifactId} diff --git a/warehouse/age-off-utils/pom.xml b/warehouse/age-off-utils/pom.xml index c4647decd0c..19f136f1b65 100644 --- a/warehouse/age-off-utils/pom.xml +++ b/warehouse/age-off-utils/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-age-off-utils ${project.artifactId} diff --git a/warehouse/age-off/pom.xml b/warehouse/age-off/pom.xml index 1afb8e56455..3f8e80e403a 100644 --- a/warehouse/age-off/pom.xml +++ b/warehouse/age-off/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-age-off ${project.artifactId} diff --git a/warehouse/assemble/datawave/pom.xml b/warehouse/assemble/datawave/pom.xml index 6a4c6d39d40..0c434e52bbc 100644 --- a/warehouse/assemble/datawave/pom.xml +++ b/warehouse/assemble/datawave/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave assemble-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 assemble-datawave pom diff --git a/warehouse/assemble/pom.xml b/warehouse/assemble/pom.xml index e9443f4ebbf..f56dcf065f6 100644 --- a/warehouse/assemble/pom.xml +++ b/warehouse/assemble/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 assemble-parent pom diff --git a/warehouse/assemble/webservice/pom.xml b/warehouse/assemble/webservice/pom.xml index 8aaef401bbb..c0b67d03abc 100644 --- a/warehouse/assemble/webservice/pom.xml +++ b/warehouse/assemble/webservice/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave assemble-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 assemble-webservice ${project.artifactId} diff --git a/warehouse/common/pom.xml b/warehouse/common/pom.xml index bfc21fa6bfc..ecb0adf9b15 100644 --- a/warehouse/common/pom.xml +++ b/warehouse/common/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-common ${project.artifactId} diff --git a/warehouse/core/pom.xml b/warehouse/core/pom.xml index 15a9d3e7343..0a3c9fde256 100644 --- a/warehouse/core/pom.xml +++ b/warehouse/core/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-core jar diff --git a/warehouse/data-dictionary-core/pom.xml b/warehouse/data-dictionary-core/pom.xml index 5c842888b10..0fd4628bb2a 100644 --- a/warehouse/data-dictionary-core/pom.xml +++ b/warehouse/data-dictionary-core/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-data-dictionary-core jar diff --git a/warehouse/edge-dictionary-core/pom.xml b/warehouse/edge-dictionary-core/pom.xml index bd582145488..b245ae92416 100644 --- a/warehouse/edge-dictionary-core/pom.xml +++ b/warehouse/edge-dictionary-core/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-edge-dictionary-core jar diff --git a/warehouse/edge-model-configuration-core/pom.xml b/warehouse/edge-model-configuration-core/pom.xml index 10182eeaa2d..84cc7fea185 100644 --- a/warehouse/edge-model-configuration-core/pom.xml +++ b/warehouse/edge-model-configuration-core/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-edge-model-configuration-core jar diff --git a/warehouse/index-stats/pom.xml b/warehouse/index-stats/pom.xml index 6aa2f1d61cb..f8881181bf1 100644 --- a/warehouse/index-stats/pom.xml +++ b/warehouse/index-stats/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-index-stats jar diff --git a/warehouse/ingest-annotation/pom.xml b/warehouse/ingest-annotation/pom.xml index f540559415d..f1ca6c22193 100644 --- a/warehouse/ingest-annotation/pom.xml +++ b/warehouse/ingest-annotation/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-ingest-annotation jar diff --git a/warehouse/ingest-configuration/pom.xml b/warehouse/ingest-configuration/pom.xml index 3a84135de41..c484dcd7425 100644 --- a/warehouse/ingest-configuration/pom.xml +++ b/warehouse/ingest-configuration/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-ingest-configuration diff --git a/warehouse/ingest-core/pom.xml b/warehouse/ingest-core/pom.xml index e140425e99a..15411c50c64 100644 --- a/warehouse/ingest-core/pom.xml +++ b/warehouse/ingest-core/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-ingest-core jar diff --git a/warehouse/ingest-csv/pom.xml b/warehouse/ingest-csv/pom.xml index f61c58dde85..b20f3fee800 100644 --- a/warehouse/ingest-csv/pom.xml +++ b/warehouse/ingest-csv/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-ingest-csv jar diff --git a/warehouse/ingest-json/pom.xml b/warehouse/ingest-json/pom.xml index 77feec02849..4d9a3f8ee71 100644 --- a/warehouse/ingest-json/pom.xml +++ b/warehouse/ingest-json/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-ingest-json jar diff --git a/warehouse/ingest-nyctlc/pom.xml b/warehouse/ingest-nyctlc/pom.xml index 6d35fca1054..53dcac366df 100644 --- a/warehouse/ingest-nyctlc/pom.xml +++ b/warehouse/ingest-nyctlc/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-ingest-nyctlc jar diff --git a/warehouse/ingest-scripts/pom.xml b/warehouse/ingest-scripts/pom.xml index c2b2c39410c..14301232ce1 100644 --- a/warehouse/ingest-scripts/pom.xml +++ b/warehouse/ingest-scripts/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-ingest-scripts ${project.artifactId} diff --git a/warehouse/ingest-ssdeep/pom.xml b/warehouse/ingest-ssdeep/pom.xml index ca038a69031..b0892a19794 100644 --- a/warehouse/ingest-ssdeep/pom.xml +++ b/warehouse/ingest-ssdeep/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-ingest-ssdeep diff --git a/warehouse/ingest-wikipedia/pom.xml b/warehouse/ingest-wikipedia/pom.xml index 4727b18796a..0668abb0fc8 100644 --- a/warehouse/ingest-wikipedia/pom.xml +++ b/warehouse/ingest-wikipedia/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-ingest-wikipedia jar diff --git a/warehouse/keyword-common/pom.xml b/warehouse/keyword-common/pom.xml index 28e13e0f316..55593114328 100644 --- a/warehouse/keyword-common/pom.xml +++ b/warehouse/keyword-common/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-keyword-common diff --git a/warehouse/metrics-core/pom.xml b/warehouse/metrics-core/pom.xml index b6ebdee709e..af859e17faf 100644 --- a/warehouse/metrics-core/pom.xml +++ b/warehouse/metrics-core/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-metrics-core jar diff --git a/warehouse/ops-tools/config-compare/pom.xml b/warehouse/ops-tools/config-compare/pom.xml index 0a025ed1d7b..42b69bf6473 100644 --- a/warehouse/ops-tools/config-compare/pom.xml +++ b/warehouse/ops-tools/config-compare/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-ops-tools-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-ops-tools-config-compare diff --git a/warehouse/ops-tools/index-validation/pom.xml b/warehouse/ops-tools/index-validation/pom.xml index c58253430e5..c04f8a6070d 100644 --- a/warehouse/ops-tools/index-validation/pom.xml +++ b/warehouse/ops-tools/index-validation/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-ops-tools-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-ops-tools-index-validation jar diff --git a/warehouse/ops-tools/pom.xml b/warehouse/ops-tools/pom.xml index f534e549a83..a545b1a2501 100644 --- a/warehouse/ops-tools/pom.xml +++ b/warehouse/ops-tools/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-ops-tools-parent pom diff --git a/warehouse/pom.xml b/warehouse/pom.xml index ce184d34e17..a4d32aef47f 100644 --- a/warehouse/pom.xml +++ b/warehouse/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-warehouse-parent pom diff --git a/warehouse/query-core/pom.xml b/warehouse/query-core/pom.xml index 8ad9ad19ed2..48022e9bb4a 100644 --- a/warehouse/query-core/pom.xml +++ b/warehouse/query-core/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-query-core jar diff --git a/warehouse/regression-testing/pom.xml b/warehouse/regression-testing/pom.xml index ac5771cbc1b..ae10e18916f 100644 --- a/warehouse/regression-testing/pom.xml +++ b/warehouse/regression-testing/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-regression-testing ${project.artifactId} diff --git a/warehouse/ssdeep-common/pom.xml b/warehouse/ssdeep-common/pom.xml index 0f91d5d7060..41f0e4f7932 100644 --- a/warehouse/ssdeep-common/pom.xml +++ b/warehouse/ssdeep-common/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-ssdeep-common diff --git a/web-services/accumulo/pom.xml b/web-services/accumulo/pom.xml index 43b66afbf27..2e0aa9178a4 100644 --- a/web-services/accumulo/pom.xml +++ b/web-services/accumulo/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-ws-accumulo ejb diff --git a/web-services/annotations/pom.xml b/web-services/annotations/pom.xml index 8f03356a001..03148c0af7b 100644 --- a/web-services/annotations/pom.xml +++ b/web-services/annotations/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-ws-annotations ejb diff --git a/web-services/atom/pom.xml b/web-services/atom/pom.xml index 10664a2801b..343c1293065 100644 --- a/web-services/atom/pom.xml +++ b/web-services/atom/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-ws-atom ejb diff --git a/web-services/cached-results/pom.xml b/web-services/cached-results/pom.xml index 93cb34d29f8..d326264bc0c 100644 --- a/web-services/cached-results/pom.xml +++ b/web-services/cached-results/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-ws-cached-results ejb diff --git a/web-services/client/pom.xml b/web-services/client/pom.xml index 96a8fd8ffe6..92917428455 100644 --- a/web-services/client/pom.xml +++ b/web-services/client/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-ws-client jar diff --git a/web-services/common-util/pom.xml b/web-services/common-util/pom.xml index 25c715365e9..4abd8941add 100644 --- a/web-services/common-util/pom.xml +++ b/web-services/common-util/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-ws-common-util jar diff --git a/web-services/common/pom.xml b/web-services/common/pom.xml index 03e9760c599..9dad82be8cf 100644 --- a/web-services/common/pom.xml +++ b/web-services/common/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-ws-common ejb diff --git a/web-services/deploy/application/pom.xml b/web-services/deploy/application/pom.xml index 96b637ba46e..3657bd1a15e 100644 --- a/web-services/deploy/application/pom.xml +++ b/web-services/deploy/application/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-deploy-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-ws-deploy-application pom diff --git a/web-services/deploy/configuration/pom.xml b/web-services/deploy/configuration/pom.xml index 41a2153ead8..f3513943672 100644 --- a/web-services/deploy/configuration/pom.xml +++ b/web-services/deploy/configuration/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-deploy-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-ws-deploy-configuration pom diff --git a/web-services/deploy/docs/pom.xml b/web-services/deploy/docs/pom.xml index 79d8db12e8d..9beb30d53a9 100644 --- a/web-services/deploy/docs/pom.xml +++ b/web-services/deploy/docs/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-deploy-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-ws-deploy-docs war diff --git a/web-services/deploy/pom.xml b/web-services/deploy/pom.xml index 74b7fc7b282..4cef0b54d75 100644 --- a/web-services/deploy/pom.xml +++ b/web-services/deploy/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 gov.nsa.datawave.webservices datawave-ws-deploy-parent diff --git a/web-services/deploy/spring-framework-integration/pom.xml b/web-services/deploy/spring-framework-integration/pom.xml index d67f5991153..95d1c8939da 100644 --- a/web-services/deploy/spring-framework-integration/pom.xml +++ b/web-services/deploy/spring-framework-integration/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-deploy-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 spring-framework-integration pom diff --git a/web-services/dictionary/pom.xml b/web-services/dictionary/pom.xml index c64dfec0b21..bb74d8ce8c7 100644 --- a/web-services/dictionary/pom.xml +++ b/web-services/dictionary/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-ws-dictionary ejb diff --git a/web-services/examples/client-login/pom.xml b/web-services/examples/client-login/pom.xml index a39299b2586..b0a47dfa321 100644 --- a/web-services/examples/client-login/pom.xml +++ b/web-services/examples/client-login/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-examples-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-ws-examples-client-login ejb diff --git a/web-services/examples/http-client/pom.xml b/web-services/examples/http-client/pom.xml index 8800b95a725..97d5964cbff 100644 --- a/web-services/examples/http-client/pom.xml +++ b/web-services/examples/http-client/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-examples-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-ws-examples-http-client jar diff --git a/web-services/examples/jms-client/pom.xml b/web-services/examples/jms-client/pom.xml index 1c572859d1c..a8dcf606c32 100644 --- a/web-services/examples/jms-client/pom.xml +++ b/web-services/examples/jms-client/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-examples-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-ws-examples-jms-client jar diff --git a/web-services/examples/pom.xml b/web-services/examples/pom.xml index bd8c128f225..73540161ad1 100644 --- a/web-services/examples/pom.xml +++ b/web-services/examples/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-ws-examples-parent pom diff --git a/web-services/examples/query-war/pom.xml b/web-services/examples/query-war/pom.xml index 685f5d81b3b..d004df14383 100644 --- a/web-services/examples/query-war/pom.xml +++ b/web-services/examples/query-war/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-examples-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-ws-examples-query-war war diff --git a/web-services/metrics/pom.xml b/web-services/metrics/pom.xml index 45802cc7391..d3ac4901e64 100644 --- a/web-services/metrics/pom.xml +++ b/web-services/metrics/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-ws-metrics ejb diff --git a/web-services/model/pom.xml b/web-services/model/pom.xml index 079d03f81e1..64dd9a8d225 100644 --- a/web-services/model/pom.xml +++ b/web-services/model/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-ws-model ejb diff --git a/web-services/modification/pom.xml b/web-services/modification/pom.xml index d0bce53cd79..341508026e8 100644 --- a/web-services/modification/pom.xml +++ b/web-services/modification/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-ws-modification ejb diff --git a/web-services/pom.xml b/web-services/pom.xml index 432656e8f8b..2aac683c7c7 100644 --- a/web-services/pom.xml +++ b/web-services/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 gov.nsa.datawave.webservices datawave-ws-parent diff --git a/web-services/query-websocket/pom.xml b/web-services/query-websocket/pom.xml index e93b23da98d..73f870ba583 100644 --- a/web-services/query-websocket/pom.xml +++ b/web-services/query-websocket/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-ws-query-websocket war diff --git a/web-services/query/pom.xml b/web-services/query/pom.xml index f9ca7002fb3..d058d1a8946 100644 --- a/web-services/query/pom.xml +++ b/web-services/query/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-ws-query ejb diff --git a/web-services/rest-api/pom.xml b/web-services/rest-api/pom.xml index 8d5254b90ef..85af68b6e53 100644 --- a/web-services/rest-api/pom.xml +++ b/web-services/rest-api/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-ws-rest-api war diff --git a/web-services/security/pom.xml b/web-services/security/pom.xml index bf1de75570a..ccf4bdf613c 100644 --- a/web-services/security/pom.xml +++ b/web-services/security/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-ws-security ejb diff --git a/web-services/web-root/pom.xml b/web-services/web-root/pom.xml index 8c5af6c1c9e..6dfdea80d35 100644 --- a/web-services/web-root/pom.xml +++ b/web-services/web-root/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.39.0-SNAPSHOT + 7.39.0-JFA-20260512-1 datawave-ws-web-root war From 49540a838c3ea9a4631ea1a8b04e39e11413251d Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Wed, 13 May 2026 09:28:30 -0400 Subject: [PATCH 54/63] fix spotbug issue --- .../datawave/query/transformer/EventQueryTransformer.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java index d294e9cbfab..c4469a7452d 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java @@ -53,6 +53,10 @@ public EventBase transform(Entry entry) { } EventBase event = this.responseObjectFactory.getEvent(); + if (null == markingFunctions) { + markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); + } + Markings markings; try { markings = this.markingFunctions.translateFromColumnVisibilityForAuths(new ColumnVisibility(key.getColumnVisibility()), this.auths); @@ -92,10 +96,6 @@ public EventBase transform(Entry entry) { // for the overall event visibility. Set visibilitiesToMerge = new HashSet<>(); - if (null == markingFunctions) { - markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); - } - for (Entry> e : eventFields.asMap().entrySet()) { origFieldName = e.getKey(); if (this.qm != null) { From 7956b996143e87ac30885d6c8979402f49ab0502 Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Wed, 13 May 2026 11:27:28 -0400 Subject: [PATCH 55/63] Jeff MR feedback round 1 --- .../cachedresults/CacheableQueryRowImpl.java | 2 - .../marking/AccessExpressionMarkings.java | 13 +++--- .../datawave/marking/MarkingFunctions.java | 6 ++- .../marking/AccessExpressionMarkingsTest.java | 13 +++--- .../MarkingFunctionsDefaultCombineTest.java | 45 +++++++++++++++++++ 5 files changed, 63 insertions(+), 16 deletions(-) diff --git a/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java b/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java index 9db486ef7ff..aed03043bb6 100644 --- a/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java +++ b/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java @@ -277,8 +277,6 @@ public void setColFam(String colFam) { } public void setMarkings(Markings markings) { - // validate the markings - markings.toAccessExpression(); this.markings = markings; } diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java index 7d0f69ef4c5..94d7c7ae655 100644 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java @@ -62,16 +62,19 @@ public boolean isEmpty() { @JsonProperty(COLUMN_VISIBILITY_KEY) @XmlElement(name = COLUMN_VISIBILITY_KEY) public String getColumnVisibilityString() { - return accessExpression != null ? accessExpression.getExpression() : ""; + return accessExpression != null ? accessExpression.getExpression() : null; } - public void setColumnVisibilityString(String expr) { - this.accessExpression = ACCESS.newExpression(expr != null ? expr : ""); + public void setColumnVisibilityString(String ae) { + this.accessExpression = ae != null ? ACCESS.newExpression(ae) : null; } @JsonCreator - public static AccessExpressionMarkings create(@JsonProperty(COLUMN_VISIBILITY_KEY) String ae) { - return AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression(ae != null ? ae : "")).build(); + public static AccessExpressionMarkings create(@JsonProperty(COLUMN_VISIBILITY_KEY) String cv) { + if (cv != null && !cv.isEmpty()) { + return AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression(cv)).build(); + } + return null; } public static AccessExpressionMarkings createMarkings(ColumnVisibility cv) { diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java index 064848ebb0a..3497ac0beec 100644 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java @@ -6,6 +6,7 @@ import java.util.Collection; import java.util.HashSet; import java.util.List; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -69,7 +70,7 @@ class Default implements MarkingFunctions { @Override public ColumnVisibility combineVisibilities(Collection visibilities) { AccessExpressionMarkings accessExpressionMarkings = combine( - visibilities.stream().map(AccessExpressionMarkings::createMarkings).collect(Collectors.toList())); + visibilities.stream().filter(Objects::nonNull).map(AccessExpressionMarkings::createMarkings).collect(Collectors.toList())); if (null == accessExpressionMarkings) { // is this correct behavior, does it match other implementations? @@ -86,6 +87,9 @@ public AccessExpressionMarkings combine(Collection> markings) { Set uniqueMarkings = new HashSet<>(); for (Markings marking : markings) { + if (null == marking) { + continue; + } if (!(marking instanceof AccessExpressionMarkings)) { throw new RuntimeException(String.format("Unknown markings class %s", marking.getClass().getName())); } diff --git a/core/utils/accumulo-utils/src/test/java/datawave/marking/AccessExpressionMarkingsTest.java b/core/utils/accumulo-utils/src/test/java/datawave/marking/AccessExpressionMarkingsTest.java index 17f5c2d7869..70677112be4 100644 --- a/core/utils/accumulo-utils/src/test/java/datawave/marking/AccessExpressionMarkingsTest.java +++ b/core/utils/accumulo-utils/src/test/java/datawave/marking/AccessExpressionMarkingsTest.java @@ -63,8 +63,7 @@ public void testIsEmptyWithNullExpression() { @Test public void testIsEmptyWithEmptyExpression() { - AccessExpressionMarkings markings = AccessExpressionMarkings.create(""); - assertTrue(markings.isEmpty()); + assertNull(AccessExpressionMarkings.create("")); } @Test @@ -84,16 +83,14 @@ public void testGetColumnVisibilityString() { @Test public void testGetColumnVisibilityStringNull() { AccessExpressionMarkings markings = AccessExpressionMarkings.builder().accessExpression(null).build(); - assertEquals("", markings.getColumnVisibilityString()); + assertNull(markings.getColumnVisibilityString()); } // -- create (JsonCreator) -- @Test public void testCreateWithNull() { - AccessExpressionMarkings markings = AccessExpressionMarkings.create(null); - assertEquals("", markings.getColumnVisibilityString()); - assertTrue(markings.isEmpty()); + assertNull(AccessExpressionMarkings.create(null)); } @Test @@ -249,7 +246,7 @@ public void testJaxbRoundTripEmpty() throws Exception { Marshaller marshaller = ctx.createMarshaller(); Unmarshaller unmarshaller = ctx.createUnmarshaller(); - AccessExpressionMarkings original = AccessExpressionMarkings.create(""); + AccessExpressionMarkings original = AccessExpressionMarkings.builder().build(); JAXBElement element = new JAXBElement<>(new QName("http://webservice.datawave.nsa/v1", "markings"), AccessExpressionMarkings.class, original); @@ -258,7 +255,7 @@ public void testJaxbRoundTripEmpty() throws Exception { JAXBElement result = unmarshaller .unmarshal(new javax.xml.transform.stream.StreamSource(new ByteArrayInputStream(out.toByteArray())), AccessExpressionMarkings.class); - assertEquals("", result.getValue().getColumnVisibilityString()); + assertNull(result.getValue().getColumnVisibilityString()); assertTrue(result.getValue().isEmpty()); } diff --git a/core/utils/accumulo-utils/src/test/java/datawave/marking/MarkingFunctionsDefaultCombineTest.java b/core/utils/accumulo-utils/src/test/java/datawave/marking/MarkingFunctionsDefaultCombineTest.java index 45af0b2a48e..bf264e5644b 100644 --- a/core/utils/accumulo-utils/src/test/java/datawave/marking/MarkingFunctionsDefaultCombineTest.java +++ b/core/utils/accumulo-utils/src/test/java/datawave/marking/MarkingFunctionsDefaultCombineTest.java @@ -2,9 +2,14 @@ import static datawave.marking.AccessExpressionMarkings.ACCESS; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.List; +import org.apache.accumulo.core.security.ColumnVisibility; import org.junit.jupiter.api.Test; public class MarkingFunctionsDefaultCombineTest { @@ -52,4 +57,44 @@ public void testCombineMapsOfOrs() throws MarkingFunctions.Exception { assertEquals(expected, markingFunctions.combine(List.of(cv1, cv2))); } + + @Test + public void testCombineSkipsNullMarkings() throws MarkingFunctions.Exception { + MarkingFunctions markingFunctions = new MarkingFunctions.Default(); + + AccessExpressionMarkings aem1 = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression("A&B")).build(); + AccessExpressionMarkings aem2 = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression("A&C")).build(); + AccessExpressionMarkings expected = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression("A&B&C")).build(); + + Collection> markings = new ArrayList<>(Arrays.asList(aem1, null, aem2, null)); + assertEquals(expected, markingFunctions.combine(markings)); + } + + @Test + public void testCombineAllNullMarkingsReturnsNull() throws MarkingFunctions.Exception { + MarkingFunctions markingFunctions = new MarkingFunctions.Default(); + + Collection> markings = new ArrayList<>(Arrays.asList(null, null)); + assertNull(markingFunctions.combine(markings)); + } + + @Test + public void testCombineVisibilitiesSkipsNullElements() throws MarkingFunctions.Exception { + MarkingFunctions markingFunctions = new MarkingFunctions.Default(); + + ColumnVisibility vis1 = new ColumnVisibility("A&B"); + ColumnVisibility vis2 = new ColumnVisibility("A&C"); + + Collection visibilities = new ArrayList<>(Arrays.asList(vis1, null, vis2, null)); + ColumnVisibility combined = markingFunctions.combineVisibilities(visibilities); + assertEquals(new ColumnVisibility("A&B&C"), combined); + } + + @Test + public void testCombineVisibilitiesAllNullsReturnsEmpty() throws MarkingFunctions.Exception { + MarkingFunctions markingFunctions = new MarkingFunctions.Default(); + + Collection visibilities = new ArrayList<>(Arrays.asList(null, null)); + assertEquals(new ColumnVisibility(), markingFunctions.combineVisibilities(visibilities)); + } } From c21f3d98873cd51ce94136092464e925067d13c9 Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Wed, 13 May 2026 11:37:39 -0400 Subject: [PATCH 56/63] Revert "use -JFA-20260512-1 modifier in POMs" This reverts commit 1158ca1d30fea089ab06243af7ed27b9bbd2407e. --- common-test/pom.xml | 2 +- contrib/datawave-quickstart/docker/pom.xml | 2 +- core/annotation/pom.xml | 2 +- core/base-rest-responses/pom.xml | 2 +- core/cached-results/pom.xml | 2 +- core/common-util/pom.xml | 2 +- core/common/pom.xml | 2 +- core/connection-pool/pom.xml | 2 +- core/map-reduce/pom.xml | 2 +- core/modification/pom.xml | 2 +- core/pom.xml | 2 +- core/query/pom.xml | 2 +- core/utils/accumulo-utils/pom.xml | 2 +- core/utils/common-utils/pom.xml | 2 +- core/utils/metadata-utils/pom.xml | 2 +- core/utils/pom.xml | 2 +- core/utils/type-utils/pom.xml | 2 +- coverage/pom.xml | 2 +- docs/pom.xml | 2 +- microservices/pom.xml | 2 +- microservices/services/accumulo/api/pom.xml | 4 ++-- .../services/accumulo/service/pom.xml | 10 +++++----- microservices/services/audit/api/pom.xml | 2 +- microservices/services/audit/service/pom.xml | 4 ++-- .../services/authorization/service/pom.xml | 6 +++--- microservices/services/dictionary/api/pom.xml | 4 ++-- .../services/dictionary/service/pom.xml | 10 +++++----- .../services/mapreduce-query/jobs/core/pom.xml | 2 +- .../services/mapreduce-query/service/pom.xml | 10 +++++----- .../services/modification/service/pom.xml | 10 +++++----- microservices/services/pom.xml | 2 +- .../services/query-executor/api/pom.xml | 6 +++--- .../services/query-executor/service/pom.xml | 16 ++++++++-------- .../services/query-metric/api/pom.xml | 6 +++--- .../services/query-metric/service/pom.xml | 16 ++++++++-------- microservices/services/query/api/pom.xml | 4 ++-- microservices/services/query/service/pom.xml | 18 +++++++++--------- microservices/starters/audit/pom.xml | 6 +++--- microservices/starters/cached-results/pom.xml | 12 ++++++------ microservices/starters/datawave/pom.xml | 4 ++-- microservices/starters/metadata/pom.xml | 6 +++--- microservices/starters/pom.xml | 2 +- microservices/starters/query-metric/pom.xml | 8 ++++---- microservices/starters/query/pom.xml | 16 ++++++++-------- pom.xml | 10 +++++----- warehouse/accumulo-extensions/pom.xml | 2 +- warehouse/age-off-utils/pom.xml | 2 +- warehouse/age-off/pom.xml | 2 +- warehouse/assemble/datawave/pom.xml | 2 +- warehouse/assemble/pom.xml | 2 +- warehouse/assemble/webservice/pom.xml | 2 +- warehouse/common/pom.xml | 2 +- warehouse/core/pom.xml | 2 +- warehouse/data-dictionary-core/pom.xml | 2 +- warehouse/edge-dictionary-core/pom.xml | 2 +- .../edge-model-configuration-core/pom.xml | 2 +- warehouse/index-stats/pom.xml | 2 +- warehouse/ingest-annotation/pom.xml | 2 +- warehouse/ingest-configuration/pom.xml | 2 +- warehouse/ingest-core/pom.xml | 2 +- warehouse/ingest-csv/pom.xml | 2 +- warehouse/ingest-json/pom.xml | 2 +- warehouse/ingest-nyctlc/pom.xml | 2 +- warehouse/ingest-scripts/pom.xml | 2 +- warehouse/ingest-ssdeep/pom.xml | 2 +- warehouse/ingest-wikipedia/pom.xml | 2 +- warehouse/keyword-common/pom.xml | 2 +- warehouse/metrics-core/pom.xml | 2 +- warehouse/ops-tools/config-compare/pom.xml | 2 +- warehouse/ops-tools/index-validation/pom.xml | 2 +- warehouse/ops-tools/pom.xml | 2 +- warehouse/pom.xml | 2 +- warehouse/query-core/pom.xml | 2 +- warehouse/regression-testing/pom.xml | 2 +- warehouse/ssdeep-common/pom.xml | 2 +- web-services/accumulo/pom.xml | 2 +- web-services/annotations/pom.xml | 2 +- web-services/atom/pom.xml | 2 +- web-services/cached-results/pom.xml | 2 +- web-services/client/pom.xml | 2 +- web-services/common-util/pom.xml | 2 +- web-services/common/pom.xml | 2 +- web-services/deploy/application/pom.xml | 2 +- web-services/deploy/configuration/pom.xml | 2 +- web-services/deploy/docs/pom.xml | 2 +- web-services/deploy/pom.xml | 2 +- .../spring-framework-integration/pom.xml | 2 +- web-services/dictionary/pom.xml | 2 +- web-services/examples/client-login/pom.xml | 2 +- web-services/examples/http-client/pom.xml | 2 +- web-services/examples/jms-client/pom.xml | 2 +- web-services/examples/pom.xml | 2 +- web-services/examples/query-war/pom.xml | 2 +- web-services/metrics/pom.xml | 2 +- web-services/model/pom.xml | 2 +- web-services/modification/pom.xml | 2 +- web-services/pom.xml | 2 +- web-services/query-websocket/pom.xml | 2 +- web-services/query/pom.xml | 2 +- web-services/rest-api/pom.xml | 2 +- web-services/security/pom.xml | 2 +- web-services/web-root/pom.xml | 2 +- 102 files changed, 174 insertions(+), 174 deletions(-) diff --git a/common-test/pom.xml b/common-test/pom.xml index 897be3e62eb..696661d047e 100644 --- a/common-test/pom.xml +++ b/common-test/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-common-test ${project.artifactId} diff --git a/contrib/datawave-quickstart/docker/pom.xml b/contrib/datawave-quickstart/docker/pom.xml index b1f13359f42..7bda04c19b0 100644 --- a/contrib/datawave-quickstart/docker/pom.xml +++ b/contrib/datawave-quickstart/docker/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT ../../../pom.xml quickstart diff --git a/core/annotation/pom.xml b/core/annotation/pom.xml index 18d753eaffd..2cf18c8c5c6 100644 --- a/core/annotation/pom.xml +++ b/core/annotation/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-core-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-core-annotation diff --git a/core/base-rest-responses/pom.xml b/core/base-rest-responses/pom.xml index 0d4a694d0ea..e5b1186af92 100644 --- a/core/base-rest-responses/pom.xml +++ b/core/base-rest-responses/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-core-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT base-rest-responses ${project.artifactId} diff --git a/core/cached-results/pom.xml b/core/cached-results/pom.xml index c366aebbf4b..c9e7a44f4f3 100644 --- a/core/cached-results/pom.xml +++ b/core/cached-results/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-core-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-core-cached-results ${project.artifactId} diff --git a/core/common-util/pom.xml b/core/common-util/pom.xml index 2a45b5960f0..5d815e50965 100644 --- a/core/common-util/pom.xml +++ b/core/common-util/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-core-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-core-common-util ${project.artifactId} diff --git a/core/common/pom.xml b/core/common/pom.xml index ae688a5b9c8..4cc11343d62 100644 --- a/core/common/pom.xml +++ b/core/common/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-core-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-core-common ${project.artifactId} diff --git a/core/connection-pool/pom.xml b/core/connection-pool/pom.xml index b4a878f5868..3eba6dba432 100644 --- a/core/connection-pool/pom.xml +++ b/core/connection-pool/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-core-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-core-connection-pool ${project.artifactId} diff --git a/core/map-reduce/pom.xml b/core/map-reduce/pom.xml index 540e3fbe736..0f6d2dc70bb 100644 --- a/core/map-reduce/pom.xml +++ b/core/map-reduce/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-core-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-core-map-reduce ${project.artifactId} diff --git a/core/modification/pom.xml b/core/modification/pom.xml index 8c65a37e854..820b45fa3ec 100644 --- a/core/modification/pom.xml +++ b/core/modification/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-core-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-core-modification ${project.artifactId} diff --git a/core/pom.xml b/core/pom.xml index 2f1e4efc57d..a2a2931c052 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT gov.nsa.datawave.core datawave-core-parent diff --git a/core/query/pom.xml b/core/query/pom.xml index c27e57b4308..9ece9a86c8e 100644 --- a/core/query/pom.xml +++ b/core/query/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-core-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-core-query ${project.artifactId} diff --git a/core/utils/accumulo-utils/pom.xml b/core/utils/accumulo-utils/pom.xml index f0db83c9df3..0d015ab8147 100644 --- a/core/utils/accumulo-utils/pom.xml +++ b/core/utils/accumulo-utils/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-utils-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT accumulo-utils ${project.artifactId} diff --git a/core/utils/common-utils/pom.xml b/core/utils/common-utils/pom.xml index a29bf4777cc..97e5de2c0ea 100644 --- a/core/utils/common-utils/pom.xml +++ b/core/utils/common-utils/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-utils-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT common-utils ${project.artifactId} diff --git a/core/utils/metadata-utils/pom.xml b/core/utils/metadata-utils/pom.xml index d679608a938..944de13daf9 100644 --- a/core/utils/metadata-utils/pom.xml +++ b/core/utils/metadata-utils/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-utils-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT metadata-utils ${project.artifactId} diff --git a/core/utils/pom.xml b/core/utils/pom.xml index 107e7127e7a..4c06579942d 100644 --- a/core/utils/pom.xml +++ b/core/utils/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-core-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT gov.nsa.datawave.core datawave-utils-parent diff --git a/core/utils/type-utils/pom.xml b/core/utils/type-utils/pom.xml index dadfca6994e..c3fe8995d22 100644 --- a/core/utils/type-utils/pom.xml +++ b/core/utils/type-utils/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-utils-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT type-utils ${project.artifactId} diff --git a/coverage/pom.xml b/coverage/pom.xml index 4991bb74183..8c691e6d7c9 100644 --- a/coverage/pom.xml +++ b/coverage/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT coverage Datawave Code Coverage Aggregate Report diff --git a/docs/pom.xml b/docs/pom.xml index 70e046c30b1..96c24783ea5 100644 --- a/docs/pom.xml +++ b/docs/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-docs diff --git a/microservices/pom.xml b/microservices/pom.xml index efc9c0d4ac1..c39adb3edb8 100644 --- a/microservices/pom.xml +++ b/microservices/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT gov.nsa.datawave.microservice datawave-microservice-build-parent diff --git a/microservices/services/accumulo/api/pom.xml b/microservices/services/accumulo/api/pom.xml index 295a9dccff7..23d1c77ad2b 100644 --- a/microservices/services/accumulo/api/pom.xml +++ b/microservices/services/accumulo/api/pom.xml @@ -8,7 +8,7 @@ ../../../microservice-parent/pom.xml accumulo-api - 4.0.2-JFA-20260512-1 + 4.0.2-SNAPSHOT https://code.nsa.gov/datawave-accumulo-service scm:git:${project.scm.url} @@ -19,7 +19,7 @@ http://webservice.datawave.nsa/v1 1.17.1 - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT 2.14.3 2.3.3 diff --git a/microservices/services/accumulo/service/pom.xml b/microservices/services/accumulo/service/pom.xml index a41e5d1ec7e..ebd73cb83a5 100644 --- a/microservices/services/accumulo/service/pom.xml +++ b/microservices/services/accumulo/service/pom.xml @@ -8,7 +8,7 @@ ../../../microservice-service-parent/pom.xml accumulo-service - 4.0.5-JFA-20260512-1 + 4.0.5-SNAPSHOT DATAWAVE Accumulo Microservice https://code.nsa.gov/datawave-accumulo-service @@ -24,12 +24,12 @@ 3.20.0 5.2.0 5.2.0 - 7.39.0-JFA-20260512-1 - 4.0.2-JFA-20260512-1 + 7.39.0-SNAPSHOT + 4.0.2-SNAPSHOT 4.0.2 4.0.2 - 4.0.8-JFA-20260512-1 - 4.0.6-JFA-20260512-1 + 4.0.8-SNAPSHOT + 4.0.6-SNAPSHOT 31.1-jre 3.3.4 4.0.4 diff --git a/microservices/services/audit/api/pom.xml b/microservices/services/audit/api/pom.xml index 63b4c953206..fa8058ce07a 100644 --- a/microservices/services/audit/api/pom.xml +++ b/microservices/services/audit/api/pom.xml @@ -8,7 +8,7 @@ ../../../microservice-parent/pom.xml audit-api - 4.0.3-JFA-20260512-1 + 4.0.3-SNAPSHOT https://code.nsa.gov/datawave-audit-service scm:git:${project.scm.url} diff --git a/microservices/services/audit/service/pom.xml b/microservices/services/audit/service/pom.xml index e951c67a9e2..445adde22fb 100644 --- a/microservices/services/audit/service/pom.xml +++ b/microservices/services/audit/service/pom.xml @@ -8,7 +8,7 @@ ../../../microservice-service-parent/pom.xml audit-service - 4.0.5-JFA-20260512-1 + 4.0.5-SNAPSHOT DATAWAVE Auditing Microservice https://code.nsa.gov/datawave-audit-service @@ -26,7 +26,7 @@ 4.0.2 4.0.4 4.0.4 - 4.0.8-JFA-20260512-1 + 4.0.8-SNAPSHOT 2.0.3 31.1-jre 3.3.4 diff --git a/microservices/services/authorization/service/pom.xml b/microservices/services/authorization/service/pom.xml index bdb479c1e10..681fe1a1a6b 100644 --- a/microservices/services/authorization/service/pom.xml +++ b/microservices/services/authorization/service/pom.xml @@ -8,7 +8,7 @@ ../../../microservice-service-parent/pom.xml authorization-service - 4.0.7-JFA-20260512-1 + 4.0.7-SNAPSHOT DATAWAVE Authorization Microservice https://code.nsa.gov/datawave-authorization-service @@ -20,11 +20,11 @@ datawave.microservice.authorization.AuthorizationService 3.20.0 - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT 4.0.2 4.0.4 4.0.4 - 4.0.8-JFA-20260512-1 + 4.0.8-SNAPSHOT 31.1-jre diff --git a/microservices/services/dictionary/api/pom.xml b/microservices/services/dictionary/api/pom.xml index 7ec8a7bf900..fa385859928 100644 --- a/microservices/services/dictionary/api/pom.xml +++ b/microservices/services/dictionary/api/pom.xml @@ -8,7 +8,7 @@ ../../../microservice-parent/pom.xml dictionary-api - 4.0.7-JFA-20260512-1 + 4.0.7-SNAPSHOT https://code.nsa.gov/datawave-dictionary-service scm:git:${project.scm.url} @@ -19,7 +19,7 @@ http://webservice.datawave.nsa/v1 3.20.0 - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT 31.1-jre 2.3.3 1.6.2 diff --git a/microservices/services/dictionary/service/pom.xml b/microservices/services/dictionary/service/pom.xml index bd175f652f1..efb4fb2f69b 100644 --- a/microservices/services/dictionary/service/pom.xml +++ b/microservices/services/dictionary/service/pom.xml @@ -8,7 +8,7 @@ ../../../microservice-service-parent/pom.xml dictionary-service - 4.0.12-JFA-20260512-1 + 4.0.12-SNAPSHOT DATAWAVE Dictionary Service https://code.nsa.gov/datawave-dictionary-service @@ -28,11 +28,11 @@ 3.2.2 3.20.0 5.2.0 - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT 4.0.2 - 4.0.7-JFA-20260512-1 - 4.0.8-JFA-20260512-1 - 3.0.5-JFA-20260512-1 + 4.0.7-SNAPSHOT + 4.0.8-SNAPSHOT + 3.0.5-SNAPSHOT 31.1-jre 3.3.4 4.0.4 diff --git a/microservices/services/mapreduce-query/jobs/core/pom.xml b/microservices/services/mapreduce-query/jobs/core/pom.xml index 12c644674ff..c4c38d98024 100644 --- a/microservices/services/mapreduce-query/jobs/core/pom.xml +++ b/microservices/services/mapreduce-query/jobs/core/pom.xml @@ -8,7 +8,7 @@ ../../../../microservice-service-parent/pom.xml mapreduce-query-core-job - 1.0.5-JFA-20260512-1 + 1.0.5-SNAPSHOT DATAWAVE MapReduce Query Core Job https://github.com/NationalSecurityAgency/datawave-mapreduce-query-service diff --git a/microservices/services/mapreduce-query/service/pom.xml b/microservices/services/mapreduce-query/service/pom.xml index 1e087a1108d..2af1396c3ce 100644 --- a/microservices/services/mapreduce-query/service/pom.xml +++ b/microservices/services/mapreduce-query/service/pom.xml @@ -8,7 +8,7 @@ ../../../microservice-service-parent/pom.xml mapreduce-query-service - 1.0.5-JFA-20260512-1 + 1.0.5-SNAPSHOT DATAWAVE MapReduce Query Microservice https://github.com/NationalSecurityAgency/datawave-mapreduce-query-service @@ -22,11 +22,11 @@ datawave/ nationalsecurityagency/ datawave.microservice.query.mapreduce.MapReduceQueryService - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT 4.0.2 - 1.0.5-JFA-20260512-1 - 4.0.8-JFA-20260512-1 - 1.0.11-JFA-20260512-1 + 1.0.5-SNAPSHOT + 4.0.8-SNAPSHOT + 1.0.11-SNAPSHOT 3.3.4 2.25.3 diff --git a/microservices/services/modification/service/pom.xml b/microservices/services/modification/service/pom.xml index f3875f54805..5614309f0b7 100644 --- a/microservices/services/modification/service/pom.xml +++ b/microservices/services/modification/service/pom.xml @@ -8,7 +8,7 @@ ../../../microservice-service-parent/pom.xml modification-service - 1.0.5-JFA-20260512-1 + 1.0.5-SNAPSHOT DATAWAVE Modification Service https://code.nsa.gov/datawave-modification-service @@ -19,12 +19,12 @@ datawave.microservice.modification.ModificationService - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT 4.0.2 1.0.1 - 1.0.2-JFA-20260512-1 - 4.0.8-JFA-20260512-1 - 3.0.5-JFA-20260512-1 + 1.0.2-SNAPSHOT + 4.0.8-SNAPSHOT + 3.0.5-SNAPSHOT 4.0.4 2.25.3 diff --git a/microservices/services/pom.xml b/microservices/services/pom.xml index 3a88185a744..e997bae4481 100644 --- a/microservices/services/pom.xml +++ b/microservices/services/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-build-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-microservice-service-build-parent pom diff --git a/microservices/services/query-executor/api/pom.xml b/microservices/services/query-executor/api/pom.xml index 57031a57eca..235ed3a37c3 100644 --- a/microservices/services/query-executor/api/pom.xml +++ b/microservices/services/query-executor/api/pom.xml @@ -8,7 +8,7 @@ ../../../microservice-parent/pom.xml query-executor-api - 1.0.10-JFA-20260512-1 + 1.0.10-SNAPSHOT https://github.com/NationalSecurityAgency/datawave-query-executor-service scm:git:${project.scm.url} @@ -19,8 +19,8 @@ http://webservice.datawave/v1 3.20.0 - 7.39.0-JFA-20260512-1 - 1.0.11-JFA-20260512-1 + 7.39.0-SNAPSHOT + 1.0.11-SNAPSHOT 2.3.3 diff --git a/microservices/services/query-executor/service/pom.xml b/microservices/services/query-executor/service/pom.xml index de79397520a..553e3654005 100644 --- a/microservices/services/query-executor/service/pom.xml +++ b/microservices/services/query-executor/service/pom.xml @@ -8,7 +8,7 @@ ../../../microservice-service-parent/pom.xml query-executor-service - 1.0.12-JFA-20260512-1 + 1.0.12-SNAPSHOT DATAWAVE Query Executor Service https://github.com/NationalSecurityAgency/datawave-query-executor-service @@ -23,14 +23,14 @@ 4.3 3.20.0 5.2.0 - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT 4.0.2 - 1.0.2-JFA-20260512-1 - 1.0.10-JFA-20260512-1 - 4.1.4-JFA-20260512-1 - 4.0.8-JFA-20260512-1 - 1.0.11-JFA-20260512-1 - 3.0.6-JFA-20260512-1 + 1.0.2-SNAPSHOT + 1.0.10-SNAPSHOT + 4.1.4-SNAPSHOT + 4.0.8-SNAPSHOT + 1.0.11-SNAPSHOT + 3.0.6-SNAPSHOT 31.1-jre 3.3.4 4.0.4 diff --git a/microservices/services/query-metric/api/pom.xml b/microservices/services/query-metric/api/pom.xml index 44956db1925..19da3c53d5c 100644 --- a/microservices/services/query-metric/api/pom.xml +++ b/microservices/services/query-metric/api/pom.xml @@ -8,7 +8,7 @@ ../../../microservice-parent/pom.xml query-metric-api - 4.1.4-JFA-20260512-1 + 4.1.4-SNAPSHOT https://code.nsa.gov/datawave-query-metric-service scm:git:${project.scm.url} @@ -21,8 +21,8 @@ ${basedir}/spotbugs-exclude.xml 3.20.0 1.9 - 7.39.0-JFA-20260512-1 - 1.0.2-JFA-20260512-1 + 7.39.0-SNAPSHOT + 1.0.2-SNAPSHOT 2.3.6 31.1-jre diff --git a/microservices/services/query-metric/service/pom.xml b/microservices/services/query-metric/service/pom.xml index 9fd498006aa..2a9f02f5a4c 100644 --- a/microservices/services/query-metric/service/pom.xml +++ b/microservices/services/query-metric/service/pom.xml @@ -8,7 +8,7 @@ ../../../microservice-service-parent/pom.xml query-metric-service - 4.1.11-JFA-20260512-1 + 4.1.11-SNAPSHOT DATAWAVE Query Metric Microservice https://code.nsa.gov/datawave-query-metric-service @@ -24,16 +24,16 @@ 1.1.1 3.3 3.20.0 - 7.39.0-JFA-20260512-1 - 4.0.2-JFA-20260512-1 + 7.39.0-SNAPSHOT + 4.0.2-SNAPSHOT 4.0.2 4.0.2 4.0.4 - 1.0.2-JFA-20260512-1 - 4.1.4-JFA-20260512-1 - 4.0.8-JFA-20260512-1 - 3.0.5-JFA-20260512-1 - 3.0.6-JFA-20260512-1 + 1.0.2-SNAPSHOT + 4.1.4-SNAPSHOT + 4.0.8-SNAPSHOT + 3.0.5-SNAPSHOT + 3.0.6-SNAPSHOT 31.1-jre 3.3.4 2.2.3 diff --git a/microservices/services/query/api/pom.xml b/microservices/services/query/api/pom.xml index b0a43187ed0..652a277eee0 100644 --- a/microservices/services/query/api/pom.xml +++ b/microservices/services/query/api/pom.xml @@ -8,7 +8,7 @@ ../../../microservice-parent/pom.xml query-api - 1.0.2-JFA-20260512-1 + 1.0.2-SNAPSHOT https://github.com/NationalSecurityAgency/datawave-query-service scm:git:${project.scm.url} @@ -20,7 +20,7 @@ http://webservice.datawave.nsa/v1 4.3 3.20.0 - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT 31.1-jre 2.14.3 diff --git a/microservices/services/query/service/pom.xml b/microservices/services/query/service/pom.xml index 4aa4635d57c..08c27501b04 100644 --- a/microservices/services/query/service/pom.xml +++ b/microservices/services/query/service/pom.xml @@ -8,7 +8,7 @@ ../../../microservice-service-parent/pom.xml query-service - 1.0.14-JFA-20260512-1 + 1.0.14-SNAPSHOT DATAWAVE Query Microservice https://github.com/NationalSecurityAgency/datawave-query-service @@ -24,17 +24,17 @@ 1.26.0 2.18.0 3.20.0 - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT 4.0.2 4.0.2 4.0.4 - 1.0.2-JFA-20260512-1 - 4.1.4-JFA-20260512-1 - 4.0.8-JFA-20260512-1 - 4.0.6-JFA-20260512-1 - 1.0.13-JFA-20260512-1 - 1.0.11-JFA-20260512-1 - 3.0.6-JFA-20260512-1 + 1.0.2-SNAPSHOT + 4.1.4-SNAPSHOT + 4.0.8-SNAPSHOT + 4.0.6-SNAPSHOT + 1.0.13-SNAPSHOT + 1.0.11-SNAPSHOT + 3.0.6-SNAPSHOT 31.1-jre 3.3.4 diff --git a/microservices/starters/audit/pom.xml b/microservices/starters/audit/pom.xml index 14a6e1d8f34..0cbc90af787 100644 --- a/microservices/starters/audit/pom.xml +++ b/microservices/starters/audit/pom.xml @@ -8,7 +8,7 @@ ../../microservice-service-parent/pom.xml spring-boot-starter-datawave-audit - 4.0.6-JFA-20260512-1 + 4.0.6-SNAPSHOT https://code.nsa.gov/datawave-spring-boot-starter-audit @@ -25,10 +25,10 @@ 1.9.25.1 3.20.0 - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT 4.0.2 4.0.2 - 4.0.8-JFA-20260512-1 + 4.0.8-SNAPSHOT 31.1-jre 2.2 diff --git a/microservices/starters/cached-results/pom.xml b/microservices/starters/cached-results/pom.xml index 8e1d56f8ad2..e5a4b5b51cb 100644 --- a/microservices/starters/cached-results/pom.xml +++ b/microservices/starters/cached-results/pom.xml @@ -8,7 +8,7 @@ ../../microservice-service-parent/pom.xml spring-boot-starter-datawave-cached-results - 1.0.13-JFA-20260512-1 + 1.0.13-SNAPSHOT https://code.nsa.gov/datawave-spring-boot-starter-cached-results @@ -25,15 +25,15 @@ 4.3 3.20.0 - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT 4.0.2 4.0.2 4.0.4 4.0.4 - 1.0.2-JFA-20260512-1 - 4.0.8-JFA-20260512-1 - 4.0.6-JFA-20260512-1 - 1.0.11-JFA-20260512-1 + 1.0.2-SNAPSHOT + 4.0.8-SNAPSHOT + 4.0.6-SNAPSHOT + 1.0.11-SNAPSHOT 9.3.0 diff --git a/microservices/starters/datawave/pom.xml b/microservices/starters/datawave/pom.xml index d79a84334e8..bc05bb9718e 100644 --- a/microservices/starters/datawave/pom.xml +++ b/microservices/starters/datawave/pom.xml @@ -8,7 +8,7 @@ ../../microservice-service-parent/pom.xml spring-boot-starter-datawave - 4.0.8-JFA-20260512-1 + 4.0.8-SNAPSHOT https://code.nsa.gov/datawave-spring-boot-starter @@ -26,7 +26,7 @@ 2.1.4-5792fed3-bulkv2 1.9.25.1 1.9.4 - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT 4.0.2 3.0.0 4.0.2 diff --git a/microservices/starters/metadata/pom.xml b/microservices/starters/metadata/pom.xml index 93d41d30ef1..ac3a09e9de3 100644 --- a/microservices/starters/metadata/pom.xml +++ b/microservices/starters/metadata/pom.xml @@ -8,7 +8,7 @@ ../../microservice-service-parent/pom.xml spring-boot-starter-datawave-metadata - 3.0.5-JFA-20260512-1 + 3.0.5-SNAPSHOT https://github.com/NationalSecurityAgency/datawave-spring-boot-starter-metadata @@ -24,8 +24,8 @@ 2.1.4-5792fed3-bulkv2 - 7.39.0-JFA-20260512-1 - 4.0.8-JFA-20260512-1 + 7.39.0-SNAPSHOT + 4.0.8-SNAPSHOT 4.0.2 2.2.42 diff --git a/microservices/starters/pom.xml b/microservices/starters/pom.xml index bac79ca2014..17aa5ff6c64 100644 --- a/microservices/starters/pom.xml +++ b/microservices/starters/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-build-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-microservice-starter-build-parent pom diff --git a/microservices/starters/query-metric/pom.xml b/microservices/starters/query-metric/pom.xml index 64ef71b7012..256125f3240 100644 --- a/microservices/starters/query-metric/pom.xml +++ b/microservices/starters/query-metric/pom.xml @@ -8,7 +8,7 @@ ../../microservice-service-parent/pom.xml spring-boot-starter-datawave-query-metric - 3.0.6-JFA-20260512-1 + 3.0.6-SNAPSHOT jar @@ -24,10 +24,10 @@ 3.20.0 - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT 4.0.2 - 4.1.4-JFA-20260512-1 - 4.0.8-JFA-20260512-1 + 4.1.4-SNAPSHOT + 4.0.8-SNAPSHOT 31.1-jre diff --git a/microservices/starters/query/pom.xml b/microservices/starters/query/pom.xml index 7a6fa068337..59463e312c6 100644 --- a/microservices/starters/query/pom.xml +++ b/microservices/starters/query/pom.xml @@ -8,7 +8,7 @@ ../../microservice-service-parent/pom.xml spring-boot-starter-datawave-query - 1.0.11-JFA-20260512-1 + 1.0.11-SNAPSHOT https://github.com/NationalSecurityAgency/datawave-spring-boot-starter-query @@ -26,18 +26,18 @@ http://webservice.datawave/v1 2.1.4-5792fed3-bulkv2 3.20.0 - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT 4.0.2 4.0.2 - 4.0.7-JFA-20260512-1 + 4.0.7-SNAPSHOT 4.0.4 4.0.4 1.0.1 - 1.0.2-JFA-20260512-1 - 4.1.4-JFA-20260512-1 - 4.0.8-JFA-20260512-1 - 3.0.5-JFA-20260512-1 - 3.0.6-JFA-20260512-1 + 1.0.2-SNAPSHOT + 4.1.4-SNAPSHOT + 4.0.8-SNAPSHOT + 3.0.5-SNAPSHOT + 3.0.6-SNAPSHOT 31.1-jre 3.3.4 diff --git a/pom.xml b/pom.xml index 28c4ec4975f..8a4b19fa174 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 gov.nsa.datawave datawave-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT pom DataWave DataWave is a Java-based ingest and query framework that leverages Apache Accumulo to provide fast, secure access to your data. @@ -75,14 +75,14 @@ 1.11.0 5.9.0 5.9.0 - 4.0.2-JFA-20260512-1 + 4.0.2-SNAPSHOT 4.0.2 4.0.2 - 4.0.7-JFA-20260512-1 + 4.0.7-SNAPSHOT 1.0.1 3.0.0 - 1.0.2-JFA-20260512-1 - 4.1.4-JFA-20260512-1 + 1.0.2-SNAPSHOT + 4.1.4-SNAPSHOT 1.9.0 3.6.3 5.2.0 diff --git a/warehouse/accumulo-extensions/pom.xml b/warehouse/accumulo-extensions/pom.xml index a8570ed0bb8..3471090617d 100644 --- a/warehouse/accumulo-extensions/pom.xml +++ b/warehouse/accumulo-extensions/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-accumulo-extensions ${project.artifactId} diff --git a/warehouse/age-off-utils/pom.xml b/warehouse/age-off-utils/pom.xml index 19f136f1b65..c4647decd0c 100644 --- a/warehouse/age-off-utils/pom.xml +++ b/warehouse/age-off-utils/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-age-off-utils ${project.artifactId} diff --git a/warehouse/age-off/pom.xml b/warehouse/age-off/pom.xml index 3f8e80e403a..1afb8e56455 100644 --- a/warehouse/age-off/pom.xml +++ b/warehouse/age-off/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-age-off ${project.artifactId} diff --git a/warehouse/assemble/datawave/pom.xml b/warehouse/assemble/datawave/pom.xml index 0c434e52bbc..6a4c6d39d40 100644 --- a/warehouse/assemble/datawave/pom.xml +++ b/warehouse/assemble/datawave/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave assemble-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT assemble-datawave pom diff --git a/warehouse/assemble/pom.xml b/warehouse/assemble/pom.xml index f56dcf065f6..e9443f4ebbf 100644 --- a/warehouse/assemble/pom.xml +++ b/warehouse/assemble/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT assemble-parent pom diff --git a/warehouse/assemble/webservice/pom.xml b/warehouse/assemble/webservice/pom.xml index c0b67d03abc..8aaef401bbb 100644 --- a/warehouse/assemble/webservice/pom.xml +++ b/warehouse/assemble/webservice/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave assemble-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT assemble-webservice ${project.artifactId} diff --git a/warehouse/common/pom.xml b/warehouse/common/pom.xml index ecb0adf9b15..bfc21fa6bfc 100644 --- a/warehouse/common/pom.xml +++ b/warehouse/common/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-common ${project.artifactId} diff --git a/warehouse/core/pom.xml b/warehouse/core/pom.xml index 0a3c9fde256..15a9d3e7343 100644 --- a/warehouse/core/pom.xml +++ b/warehouse/core/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-core jar diff --git a/warehouse/data-dictionary-core/pom.xml b/warehouse/data-dictionary-core/pom.xml index 0fd4628bb2a..5c842888b10 100644 --- a/warehouse/data-dictionary-core/pom.xml +++ b/warehouse/data-dictionary-core/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-data-dictionary-core jar diff --git a/warehouse/edge-dictionary-core/pom.xml b/warehouse/edge-dictionary-core/pom.xml index b245ae92416..bd582145488 100644 --- a/warehouse/edge-dictionary-core/pom.xml +++ b/warehouse/edge-dictionary-core/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-edge-dictionary-core jar diff --git a/warehouse/edge-model-configuration-core/pom.xml b/warehouse/edge-model-configuration-core/pom.xml index 84cc7fea185..10182eeaa2d 100644 --- a/warehouse/edge-model-configuration-core/pom.xml +++ b/warehouse/edge-model-configuration-core/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-edge-model-configuration-core jar diff --git a/warehouse/index-stats/pom.xml b/warehouse/index-stats/pom.xml index f8881181bf1..6aa2f1d61cb 100644 --- a/warehouse/index-stats/pom.xml +++ b/warehouse/index-stats/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-index-stats jar diff --git a/warehouse/ingest-annotation/pom.xml b/warehouse/ingest-annotation/pom.xml index f1ca6c22193..f540559415d 100644 --- a/warehouse/ingest-annotation/pom.xml +++ b/warehouse/ingest-annotation/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-ingest-annotation jar diff --git a/warehouse/ingest-configuration/pom.xml b/warehouse/ingest-configuration/pom.xml index c484dcd7425..3a84135de41 100644 --- a/warehouse/ingest-configuration/pom.xml +++ b/warehouse/ingest-configuration/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-ingest-configuration diff --git a/warehouse/ingest-core/pom.xml b/warehouse/ingest-core/pom.xml index 15411c50c64..e140425e99a 100644 --- a/warehouse/ingest-core/pom.xml +++ b/warehouse/ingest-core/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-ingest-core jar diff --git a/warehouse/ingest-csv/pom.xml b/warehouse/ingest-csv/pom.xml index b20f3fee800..f61c58dde85 100644 --- a/warehouse/ingest-csv/pom.xml +++ b/warehouse/ingest-csv/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-ingest-csv jar diff --git a/warehouse/ingest-json/pom.xml b/warehouse/ingest-json/pom.xml index 4d9a3f8ee71..77feec02849 100644 --- a/warehouse/ingest-json/pom.xml +++ b/warehouse/ingest-json/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-ingest-json jar diff --git a/warehouse/ingest-nyctlc/pom.xml b/warehouse/ingest-nyctlc/pom.xml index 53dcac366df..6d35fca1054 100644 --- a/warehouse/ingest-nyctlc/pom.xml +++ b/warehouse/ingest-nyctlc/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-ingest-nyctlc jar diff --git a/warehouse/ingest-scripts/pom.xml b/warehouse/ingest-scripts/pom.xml index 14301232ce1..c2b2c39410c 100644 --- a/warehouse/ingest-scripts/pom.xml +++ b/warehouse/ingest-scripts/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-ingest-scripts ${project.artifactId} diff --git a/warehouse/ingest-ssdeep/pom.xml b/warehouse/ingest-ssdeep/pom.xml index b0892a19794..ca038a69031 100644 --- a/warehouse/ingest-ssdeep/pom.xml +++ b/warehouse/ingest-ssdeep/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-ingest-ssdeep diff --git a/warehouse/ingest-wikipedia/pom.xml b/warehouse/ingest-wikipedia/pom.xml index 0668abb0fc8..4727b18796a 100644 --- a/warehouse/ingest-wikipedia/pom.xml +++ b/warehouse/ingest-wikipedia/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-ingest-wikipedia jar diff --git a/warehouse/keyword-common/pom.xml b/warehouse/keyword-common/pom.xml index 55593114328..28e13e0f316 100644 --- a/warehouse/keyword-common/pom.xml +++ b/warehouse/keyword-common/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-keyword-common diff --git a/warehouse/metrics-core/pom.xml b/warehouse/metrics-core/pom.xml index af859e17faf..b6ebdee709e 100644 --- a/warehouse/metrics-core/pom.xml +++ b/warehouse/metrics-core/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-metrics-core jar diff --git a/warehouse/ops-tools/config-compare/pom.xml b/warehouse/ops-tools/config-compare/pom.xml index 42b69bf6473..0a025ed1d7b 100644 --- a/warehouse/ops-tools/config-compare/pom.xml +++ b/warehouse/ops-tools/config-compare/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-ops-tools-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-ops-tools-config-compare diff --git a/warehouse/ops-tools/index-validation/pom.xml b/warehouse/ops-tools/index-validation/pom.xml index c04f8a6070d..c58253430e5 100644 --- a/warehouse/ops-tools/index-validation/pom.xml +++ b/warehouse/ops-tools/index-validation/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-ops-tools-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-ops-tools-index-validation jar diff --git a/warehouse/ops-tools/pom.xml b/warehouse/ops-tools/pom.xml index a545b1a2501..f534e549a83 100644 --- a/warehouse/ops-tools/pom.xml +++ b/warehouse/ops-tools/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-ops-tools-parent pom diff --git a/warehouse/pom.xml b/warehouse/pom.xml index a4d32aef47f..ce184d34e17 100644 --- a/warehouse/pom.xml +++ b/warehouse/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-warehouse-parent pom diff --git a/warehouse/query-core/pom.xml b/warehouse/query-core/pom.xml index 48022e9bb4a..8ad9ad19ed2 100644 --- a/warehouse/query-core/pom.xml +++ b/warehouse/query-core/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-query-core jar diff --git a/warehouse/regression-testing/pom.xml b/warehouse/regression-testing/pom.xml index ae10e18916f..ac5771cbc1b 100644 --- a/warehouse/regression-testing/pom.xml +++ b/warehouse/regression-testing/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-regression-testing ${project.artifactId} diff --git a/warehouse/ssdeep-common/pom.xml b/warehouse/ssdeep-common/pom.xml index 41f0e4f7932..0f91d5d7060 100644 --- a/warehouse/ssdeep-common/pom.xml +++ b/warehouse/ssdeep-common/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-ssdeep-common diff --git a/web-services/accumulo/pom.xml b/web-services/accumulo/pom.xml index 2e0aa9178a4..43b66afbf27 100644 --- a/web-services/accumulo/pom.xml +++ b/web-services/accumulo/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-ws-accumulo ejb diff --git a/web-services/annotations/pom.xml b/web-services/annotations/pom.xml index 03148c0af7b..8f03356a001 100644 --- a/web-services/annotations/pom.xml +++ b/web-services/annotations/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-ws-annotations ejb diff --git a/web-services/atom/pom.xml b/web-services/atom/pom.xml index 343c1293065..10664a2801b 100644 --- a/web-services/atom/pom.xml +++ b/web-services/atom/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-ws-atom ejb diff --git a/web-services/cached-results/pom.xml b/web-services/cached-results/pom.xml index d326264bc0c..93cb34d29f8 100644 --- a/web-services/cached-results/pom.xml +++ b/web-services/cached-results/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-ws-cached-results ejb diff --git a/web-services/client/pom.xml b/web-services/client/pom.xml index 92917428455..96a8fd8ffe6 100644 --- a/web-services/client/pom.xml +++ b/web-services/client/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-ws-client jar diff --git a/web-services/common-util/pom.xml b/web-services/common-util/pom.xml index 4abd8941add..25c715365e9 100644 --- a/web-services/common-util/pom.xml +++ b/web-services/common-util/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-ws-common-util jar diff --git a/web-services/common/pom.xml b/web-services/common/pom.xml index 9dad82be8cf..03e9760c599 100644 --- a/web-services/common/pom.xml +++ b/web-services/common/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-ws-common ejb diff --git a/web-services/deploy/application/pom.xml b/web-services/deploy/application/pom.xml index 3657bd1a15e..96b637ba46e 100644 --- a/web-services/deploy/application/pom.xml +++ b/web-services/deploy/application/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-deploy-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-ws-deploy-application pom diff --git a/web-services/deploy/configuration/pom.xml b/web-services/deploy/configuration/pom.xml index f3513943672..41a2153ead8 100644 --- a/web-services/deploy/configuration/pom.xml +++ b/web-services/deploy/configuration/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-deploy-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-ws-deploy-configuration pom diff --git a/web-services/deploy/docs/pom.xml b/web-services/deploy/docs/pom.xml index 9beb30d53a9..79d8db12e8d 100644 --- a/web-services/deploy/docs/pom.xml +++ b/web-services/deploy/docs/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-deploy-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-ws-deploy-docs war diff --git a/web-services/deploy/pom.xml b/web-services/deploy/pom.xml index 4cef0b54d75..74b7fc7b282 100644 --- a/web-services/deploy/pom.xml +++ b/web-services/deploy/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT gov.nsa.datawave.webservices datawave-ws-deploy-parent diff --git a/web-services/deploy/spring-framework-integration/pom.xml b/web-services/deploy/spring-framework-integration/pom.xml index 95d1c8939da..d67f5991153 100644 --- a/web-services/deploy/spring-framework-integration/pom.xml +++ b/web-services/deploy/spring-framework-integration/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-deploy-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT spring-framework-integration pom diff --git a/web-services/dictionary/pom.xml b/web-services/dictionary/pom.xml index bb74d8ce8c7..c64dfec0b21 100644 --- a/web-services/dictionary/pom.xml +++ b/web-services/dictionary/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-ws-dictionary ejb diff --git a/web-services/examples/client-login/pom.xml b/web-services/examples/client-login/pom.xml index b0a47dfa321..a39299b2586 100644 --- a/web-services/examples/client-login/pom.xml +++ b/web-services/examples/client-login/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-examples-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-ws-examples-client-login ejb diff --git a/web-services/examples/http-client/pom.xml b/web-services/examples/http-client/pom.xml index 97d5964cbff..8800b95a725 100644 --- a/web-services/examples/http-client/pom.xml +++ b/web-services/examples/http-client/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-examples-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-ws-examples-http-client jar diff --git a/web-services/examples/jms-client/pom.xml b/web-services/examples/jms-client/pom.xml index a8dcf606c32..1c572859d1c 100644 --- a/web-services/examples/jms-client/pom.xml +++ b/web-services/examples/jms-client/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-examples-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-ws-examples-jms-client jar diff --git a/web-services/examples/pom.xml b/web-services/examples/pom.xml index 73540161ad1..bd8c128f225 100644 --- a/web-services/examples/pom.xml +++ b/web-services/examples/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-ws-examples-parent pom diff --git a/web-services/examples/query-war/pom.xml b/web-services/examples/query-war/pom.xml index d004df14383..685f5d81b3b 100644 --- a/web-services/examples/query-war/pom.xml +++ b/web-services/examples/query-war/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-examples-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-ws-examples-query-war war diff --git a/web-services/metrics/pom.xml b/web-services/metrics/pom.xml index d3ac4901e64..45802cc7391 100644 --- a/web-services/metrics/pom.xml +++ b/web-services/metrics/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-ws-metrics ejb diff --git a/web-services/model/pom.xml b/web-services/model/pom.xml index 64dd9a8d225..079d03f81e1 100644 --- a/web-services/model/pom.xml +++ b/web-services/model/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-ws-model ejb diff --git a/web-services/modification/pom.xml b/web-services/modification/pom.xml index 341508026e8..d0bce53cd79 100644 --- a/web-services/modification/pom.xml +++ b/web-services/modification/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-ws-modification ejb diff --git a/web-services/pom.xml b/web-services/pom.xml index 2aac683c7c7..432656e8f8b 100644 --- a/web-services/pom.xml +++ b/web-services/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT gov.nsa.datawave.webservices datawave-ws-parent diff --git a/web-services/query-websocket/pom.xml b/web-services/query-websocket/pom.xml index 73f870ba583..e93b23da98d 100644 --- a/web-services/query-websocket/pom.xml +++ b/web-services/query-websocket/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-ws-query-websocket war diff --git a/web-services/query/pom.xml b/web-services/query/pom.xml index d058d1a8946..f9ca7002fb3 100644 --- a/web-services/query/pom.xml +++ b/web-services/query/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-ws-query ejb diff --git a/web-services/rest-api/pom.xml b/web-services/rest-api/pom.xml index 85af68b6e53..8d5254b90ef 100644 --- a/web-services/rest-api/pom.xml +++ b/web-services/rest-api/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-ws-rest-api war diff --git a/web-services/security/pom.xml b/web-services/security/pom.xml index ccf4bdf613c..bf1de75570a 100644 --- a/web-services/security/pom.xml +++ b/web-services/security/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-ws-security ejb diff --git a/web-services/web-root/pom.xml b/web-services/web-root/pom.xml index 6dfdea80d35..8c5af6c1c9e 100644 --- a/web-services/web-root/pom.xml +++ b/web-services/web-root/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.39.0-JFA-20260512-1 + 7.39.0-SNAPSHOT datawave-ws-web-root war From c07701b59305429554e30c2df68bd7a241b3c180 Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Thu, 14 May 2026 12:27:49 -0400 Subject: [PATCH 57/63] fix XmlElement usage in DefaultEvent and DefaultField for markings --- .../datawave/webservice/query/result/event/DefaultEvent.java | 3 +-- .../datawave/webservice/query/result/event/DefaultField.java | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultEvent.java b/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultEvent.java index cee73ef751d..837c8d8d38f 100644 --- a/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultEvent.java +++ b/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultEvent.java @@ -30,7 +30,7 @@ public class DefaultEvent extends EventBase implement private static final long serialVersionUID = 2390592143914560317L; - @XmlElement(name = "Markings") + @XmlElement(name = "Markings", type = AccessExpressionMarkings.class) private Markings markings; @XmlElement(name = "Metadata") @@ -54,7 +54,6 @@ public String toString() { } @Override - @XmlElement(name = "Markings", type = AccessExpressionMarkings.class) public Markings getMarkings() { return this.markings; } diff --git a/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultField.java b/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultField.java index 21c66cde6bd..0acf4b41fc3 100644 --- a/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultField.java +++ b/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultField.java @@ -37,7 +37,7 @@ public class DefaultField extends FieldBase implements Serializabl private static final long serialVersionUID = -3982566563059126017L; - @XmlElement(name = "Markings") + @XmlElement(name = "Markings", type = AccessExpressionMarkings.class) private Markings markings; @XmlAttribute(name = "columnVisibility") private String columnVisibility; From 9065fc83160d070a2310c0942702f90358fa95b9 Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Thu, 14 May 2026 15:55:56 -0400 Subject: [PATCH 58/63] Fix: Handle null ColumnVisibility in ModelKeyParser --- .../java/datawave/marking/AccessExpressionMarkings.java | 1 + .../main/java/datawave/query/model/ModelKeyParser.java | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java index 94d7c7ae655..5d67619b67b 100644 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java @@ -65,6 +65,7 @@ public String getColumnVisibilityString() { return accessExpression != null ? accessExpression.getExpression() : null; } + // required for JAXB unmarshalling public void setColumnVisibilityString(String ae) { this.accessExpression = ae != null ? ACCESS.newExpression(ae) : null; } diff --git a/core/utils/metadata-utils/src/main/java/datawave/query/model/ModelKeyParser.java b/core/utils/metadata-utils/src/main/java/datawave/query/model/ModelKeyParser.java index 1a4f9a01a9b..f2521e31175 100644 --- a/core/utils/metadata-utils/src/main/java/datawave/query/model/ModelKeyParser.java +++ b/core/utils/metadata-utils/src/main/java/datawave/query/model/ModelKeyParser.java @@ -127,6 +127,9 @@ public static FieldMapping parseKey(Key key, Value value) { } public static Key createKey(FieldMapping mapping, String modelName) { + if (null == mapping.getColumnVisibility()) { + throw new RuntimeException("ColumnVisibility was null, unable to create Accumulo mutation"); + } ColumnVisibility cv = new ColumnVisibility(mapping.getColumnVisibility()); String dataType = StringUtils.isEmpty(mapping.getDatatype()) ? "" : NULL_BYTE + mapping.getDatatype().trim(); @@ -185,6 +188,9 @@ private static Set getAttrValues(Set attributes) { } public static Mutation createMutation(FieldMapping mapping, String modelName) { + if (null == mapping.getColumnVisibility()) { + throw new RuntimeException("ColumnVisibility was null, unable to create Accumulo mutation"); + } ColumnVisibility cv = new ColumnVisibility(mapping.getColumnVisibility()); Mutation m; String dataType = StringUtils.isEmpty(mapping.getDatatype()) ? "" : NULL_BYTE + mapping.getDatatype().trim(); @@ -208,6 +214,9 @@ public static Mutation createMutation(FieldMapping mapping, String modelName) { } public static Mutation createDeleteMutation(FieldMapping mapping, String modelName) { + if (null == mapping.getColumnVisibility()) { + throw new RuntimeException("ColumnVisibility was null, unable to create Accumulo mutation"); + } ColumnVisibility cv = new ColumnVisibility(mapping.getColumnVisibility()); Mutation m; String dataType = StringUtils.isEmpty(mapping.getDatatype()) ? "" : NULL_BYTE + mapping.getDatatype().trim(); From 7be1fe8f5e955940bff3f5834f1cbaa67bc163a2 Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Fri, 15 May 2026 10:08:05 -0400 Subject: [PATCH 59/63] Fix: Handle null/empty ColumnVisibility in MetadataDescriptionsHelper --- .../microservice/metadata/MetadataDescriptionsHelper.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java index c9ce0753a45..8206e04b59c 100644 --- a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java +++ b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java @@ -127,6 +127,9 @@ public void setDescriptions(MetadataEntry entry, Set if (desc.getMarkings() != null) { cv = desc.getMarkings().toColumnVisibility(); } + if (null == cv || cv.getExpression().length == 0) { + throw new IllegalArgumentException("ColumnVisibility was null or empty, unable to create Accumulo mutation"); + } m.put(ColumnFamilyConstants.COLF_DESC, new Text(entry.getDatatype()), cv, new Value(desc.getDescription().getBytes(UTF_8))); } bw.addMutation(m); @@ -163,6 +166,9 @@ public void removeDescription(MetadataEntry entry, DescriptionBase desc) throws if (desc.getMarkings() != null) { cv = desc.getMarkings().toColumnVisibility(); } + if (null == cv || cv.getExpression().length == 0) { + throw new IllegalArgumentException("ColumnVisibility was null or empty, unable to create Accumulo mutation"); + } m.putDelete(ColumnFamilyConstants.COLF_DESC, new Text(entry.getDatatype()), cv); bw.addMutation(m); } finally { From dc20c60673b015449132eaa0dfbcf6acd5901a38 Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Fri, 15 May 2026 15:12:16 -0400 Subject: [PATCH 60/63] update accumulo-access-core version --- core/utils/accumulo-utils/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/utils/accumulo-utils/pom.xml b/core/utils/accumulo-utils/pom.xml index 0d015ab8147..4c8ff48b870 100644 --- a/core/utils/accumulo-utils/pom.xml +++ b/core/utils/accumulo-utils/pom.xml @@ -70,7 +70,7 @@ org.apache.accumulo accumulo-access-core - 1.0.0-java-11 + 1.0.0-beta3 org.apache.commons From 3d8ab21487dd505ce94e9cf2a429bbf5cf05961b Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Thu, 21 May 2026 12:54:49 -0400 Subject: [PATCH 61/63] reset microservice versions that did not change --- microservices/services/audit/service/pom.xml | 2 +- .../services/authorization/service/pom.xml | 4 ++-- microservices/services/dictionary/api/pom.xml | 9 +++++++++ .../services/mapreduce-query/service/pom.xml | 6 +++--- .../services/modification/service/pom.xml | 8 ++++---- microservices/services/query-executor/api/pom.xml | 4 ++-- .../services/query-executor/service/pom.xml | 14 +++++++------- microservices/services/query-metric/api/pom.xml | 2 +- .../services/query-metric/service/pom.xml | 2 +- microservices/services/query/api/pom.xml | 2 +- microservices/services/query/service/pom.xml | 11 ++++++++++- microservices/starters/cached-results/pom.xml | 2 +- microservices/starters/query/pom.xml | 2 +- pom.xml | 2 +- 14 files changed, 44 insertions(+), 26 deletions(-) diff --git a/microservices/services/audit/service/pom.xml b/microservices/services/audit/service/pom.xml index 445adde22fb..cfdab2d5c54 100644 --- a/microservices/services/audit/service/pom.xml +++ b/microservices/services/audit/service/pom.xml @@ -26,7 +26,7 @@ 4.0.2 4.0.4 4.0.4 - 4.0.8-SNAPSHOT + 4.0.7 2.0.3 31.1-jre 3.3.4 diff --git a/microservices/services/authorization/service/pom.xml b/microservices/services/authorization/service/pom.xml index 681fe1a1a6b..361a9c6624b 100644 --- a/microservices/services/authorization/service/pom.xml +++ b/microservices/services/authorization/service/pom.xml @@ -20,11 +20,11 @@ datawave.microservice.authorization.AuthorizationService 3.20.0 - 7.39.0-SNAPSHOT + 7.33.1 4.0.2 4.0.4 4.0.4 - 4.0.8-SNAPSHOT + 4.0.7 31.1-jre diff --git a/microservices/services/dictionary/api/pom.xml b/microservices/services/dictionary/api/pom.xml index fa385859928..e7db03f4446 100644 --- a/microservices/services/dictionary/api/pom.xml +++ b/microservices/services/dictionary/api/pom.xml @@ -31,6 +31,11 @@ guava ${version.guava} + + gov.nsa.datawave.core + accumulo-utils + ${version.datawave} + gov.nsa.datawave.core base-rest-responses @@ -69,6 +74,10 @@ com.google.guava guava + + gov.nsa.datawave.core + accumulo-utils + gov.nsa.datawave.core base-rest-responses diff --git a/microservices/services/mapreduce-query/service/pom.xml b/microservices/services/mapreduce-query/service/pom.xml index 2af1396c3ce..672dbcb0ca4 100644 --- a/microservices/services/mapreduce-query/service/pom.xml +++ b/microservices/services/mapreduce-query/service/pom.xml @@ -22,11 +22,11 @@ datawave/ nationalsecurityagency/ datawave.microservice.query.mapreduce.MapReduceQueryService - 7.39.0-SNAPSHOT + 7.33.1 4.0.2 1.0.5-SNAPSHOT - 4.0.8-SNAPSHOT - 1.0.11-SNAPSHOT + 4.0.7 + 1.0.10 3.3.4 2.25.3 diff --git a/microservices/services/modification/service/pom.xml b/microservices/services/modification/service/pom.xml index 5614309f0b7..14c64ee1816 100644 --- a/microservices/services/modification/service/pom.xml +++ b/microservices/services/modification/service/pom.xml @@ -19,12 +19,12 @@ datawave.microservice.modification.ModificationService - 7.39.0-SNAPSHOT + 7.33.1 4.0.2 1.0.1 - 1.0.2-SNAPSHOT - 4.0.8-SNAPSHOT - 3.0.5-SNAPSHOT + 1.0.1 + 4.0.7 + 3.0.4 4.0.4 2.25.3 diff --git a/microservices/services/query-executor/api/pom.xml b/microservices/services/query-executor/api/pom.xml index 235ed3a37c3..84313acbedf 100644 --- a/microservices/services/query-executor/api/pom.xml +++ b/microservices/services/query-executor/api/pom.xml @@ -19,8 +19,8 @@ http://webservice.datawave/v1 3.20.0 - 7.39.0-SNAPSHOT - 1.0.11-SNAPSHOT + 7.33.1 + 1.0.10 2.3.3 diff --git a/microservices/services/query-executor/service/pom.xml b/microservices/services/query-executor/service/pom.xml index 553e3654005..bc7cd01bef2 100644 --- a/microservices/services/query-executor/service/pom.xml +++ b/microservices/services/query-executor/service/pom.xml @@ -23,14 +23,14 @@ 4.3 3.20.0 5.2.0 - 7.39.0-SNAPSHOT + 7.33.1 4.0.2 - 1.0.2-SNAPSHOT - 1.0.10-SNAPSHOT - 4.1.4-SNAPSHOT - 4.0.8-SNAPSHOT - 1.0.11-SNAPSHOT - 3.0.6-SNAPSHOT + 1.0.1 + 1.0.9 + 4.1.3 + 4.0.7 + 1.0.10 + 3.0.5 31.1-jre 3.3.4 4.0.4 diff --git a/microservices/services/query-metric/api/pom.xml b/microservices/services/query-metric/api/pom.xml index 19da3c53d5c..b3d5e7e1c91 100644 --- a/microservices/services/query-metric/api/pom.xml +++ b/microservices/services/query-metric/api/pom.xml @@ -22,7 +22,7 @@ 3.20.0 1.9 7.39.0-SNAPSHOT - 1.0.2-SNAPSHOT + 1.0.1 2.3.6 31.1-jre diff --git a/microservices/services/query-metric/service/pom.xml b/microservices/services/query-metric/service/pom.xml index 2a9f02f5a4c..f29474ed61a 100644 --- a/microservices/services/query-metric/service/pom.xml +++ b/microservices/services/query-metric/service/pom.xml @@ -29,7 +29,7 @@ 4.0.2 4.0.2 4.0.4 - 1.0.2-SNAPSHOT + 1.0.1 4.1.4-SNAPSHOT 4.0.8-SNAPSHOT 3.0.5-SNAPSHOT diff --git a/microservices/services/query/api/pom.xml b/microservices/services/query/api/pom.xml index 652a277eee0..21b88bf96b2 100644 --- a/microservices/services/query/api/pom.xml +++ b/microservices/services/query/api/pom.xml @@ -20,7 +20,7 @@ http://webservice.datawave.nsa/v1 4.3 3.20.0 - 7.39.0-SNAPSHOT + 7.33.1 31.1-jre 2.14.3 diff --git a/microservices/services/query/service/pom.xml b/microservices/services/query/service/pom.xml index 08c27501b04..9ed9fcf338c 100644 --- a/microservices/services/query/service/pom.xml +++ b/microservices/services/query/service/pom.xml @@ -28,7 +28,7 @@ 4.0.2 4.0.2 4.0.4 - 1.0.2-SNAPSHOT + 1.0.1 4.1.4-SNAPSHOT 4.0.8-SNAPSHOT 4.0.6-SNAPSHOT @@ -75,6 +75,11 @@ datawave-query-core ${version.datawave} + + gov.nsa.datawave.core + accumulo-utils + ${version.datawave} + gov.nsa.datawave.core base-rest-responses @@ -367,6 +372,10 @@ + + gov.nsa.datawave.core + accumulo-utils + gov.nsa.datawave.core base-rest-responses diff --git a/microservices/starters/cached-results/pom.xml b/microservices/starters/cached-results/pom.xml index e5a4b5b51cb..e10db7d9328 100644 --- a/microservices/starters/cached-results/pom.xml +++ b/microservices/starters/cached-results/pom.xml @@ -30,7 +30,7 @@ 4.0.2 4.0.4 4.0.4 - 1.0.2-SNAPSHOT + 1.0.1 4.0.8-SNAPSHOT 4.0.6-SNAPSHOT 1.0.11-SNAPSHOT diff --git a/microservices/starters/query/pom.xml b/microservices/starters/query/pom.xml index 59463e312c6..2c6cae161c9 100644 --- a/microservices/starters/query/pom.xml +++ b/microservices/starters/query/pom.xml @@ -33,7 +33,7 @@ 4.0.4 4.0.4 1.0.1 - 1.0.2-SNAPSHOT + 1.0.1 4.1.4-SNAPSHOT 4.0.8-SNAPSHOT 3.0.5-SNAPSHOT diff --git a/pom.xml b/pom.xml index 8a4b19fa174..c99d9bf5741 100644 --- a/pom.xml +++ b/pom.xml @@ -81,7 +81,7 @@ 4.0.7-SNAPSHOT 1.0.1 3.0.0 - 1.0.2-SNAPSHOT + 1.0.1 4.1.4-SNAPSHOT 1.9.0 3.6.3 From fd2c422fc8a2da134ed450d716e0340e9012bd76 Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Thu, 21 May 2026 14:24:31 -0400 Subject: [PATCH 62/63] reset starters that did not need to change --- microservices/services/dictionary/service/pom.xml | 2 +- microservices/services/query-metric/service/pom.xml | 4 ++-- microservices/services/query/service/pom.xml | 4 ++-- microservices/starters/cached-results/pom.xml | 2 +- microservices/starters/metadata/pom.xml | 4 ++-- microservices/starters/query-metric/pom.xml | 6 +++--- microservices/starters/query/pom.xml | 12 ++++++------ 7 files changed, 17 insertions(+), 17 deletions(-) diff --git a/microservices/services/dictionary/service/pom.xml b/microservices/services/dictionary/service/pom.xml index efb4fb2f69b..d48167e77b8 100644 --- a/microservices/services/dictionary/service/pom.xml +++ b/microservices/services/dictionary/service/pom.xml @@ -32,7 +32,7 @@ 4.0.2 4.0.7-SNAPSHOT 4.0.8-SNAPSHOT - 3.0.5-SNAPSHOT + 3.0.4 31.1-jre 3.3.4 4.0.4 diff --git a/microservices/services/query-metric/service/pom.xml b/microservices/services/query-metric/service/pom.xml index f29474ed61a..3f3b7aff413 100644 --- a/microservices/services/query-metric/service/pom.xml +++ b/microservices/services/query-metric/service/pom.xml @@ -32,8 +32,8 @@ 1.0.1 4.1.4-SNAPSHOT 4.0.8-SNAPSHOT - 3.0.5-SNAPSHOT - 3.0.6-SNAPSHOT + 3.0.4 + 3.0.5 31.1-jre 3.3.4 2.2.3 diff --git a/microservices/services/query/service/pom.xml b/microservices/services/query/service/pom.xml index 9ed9fcf338c..149d79661c3 100644 --- a/microservices/services/query/service/pom.xml +++ b/microservices/services/query/service/pom.xml @@ -33,8 +33,8 @@ 4.0.8-SNAPSHOT 4.0.6-SNAPSHOT 1.0.13-SNAPSHOT - 1.0.11-SNAPSHOT - 3.0.6-SNAPSHOT + 1.0.10 + 3.0.5 31.1-jre 3.3.4 diff --git a/microservices/starters/cached-results/pom.xml b/microservices/starters/cached-results/pom.xml index e10db7d9328..3653e7a04fc 100644 --- a/microservices/starters/cached-results/pom.xml +++ b/microservices/starters/cached-results/pom.xml @@ -33,7 +33,7 @@ 1.0.1 4.0.8-SNAPSHOT 4.0.6-SNAPSHOT - 1.0.11-SNAPSHOT + 1.0.10 9.3.0 diff --git a/microservices/starters/metadata/pom.xml b/microservices/starters/metadata/pom.xml index ac3a09e9de3..0fe00a2e34a 100644 --- a/microservices/starters/metadata/pom.xml +++ b/microservices/starters/metadata/pom.xml @@ -24,8 +24,8 @@ 2.1.4-5792fed3-bulkv2 - 7.39.0-SNAPSHOT - 4.0.8-SNAPSHOT + 7.33.1 + 4.0.7 4.0.2 2.2.42 diff --git a/microservices/starters/query-metric/pom.xml b/microservices/starters/query-metric/pom.xml index 256125f3240..551b6b71b12 100644 --- a/microservices/starters/query-metric/pom.xml +++ b/microservices/starters/query-metric/pom.xml @@ -24,10 +24,10 @@ 3.20.0 - 7.39.0-SNAPSHOT + 7.33.1 4.0.2 - 4.1.4-SNAPSHOT - 4.0.8-SNAPSHOT + 4.1.3 + 4.0.7 31.1-jre diff --git a/microservices/starters/query/pom.xml b/microservices/starters/query/pom.xml index 2c6cae161c9..628f46ffeee 100644 --- a/microservices/starters/query/pom.xml +++ b/microservices/starters/query/pom.xml @@ -26,18 +26,18 @@ http://webservice.datawave/v1 2.1.4-5792fed3-bulkv2 3.20.0 - 7.39.0-SNAPSHOT + 7.33.1 4.0.2 4.0.2 - 4.0.7-SNAPSHOT + 4.0.6 4.0.4 4.0.4 1.0.1 1.0.1 - 4.1.4-SNAPSHOT - 4.0.8-SNAPSHOT - 3.0.5-SNAPSHOT - 3.0.6-SNAPSHOT + 4.1.3 + 4.0.7 + 3.0.4 + 3.0.5 31.1-jre 3.3.4 From de9dd3daa728e54427598d163897b779a4889fcd Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Thu, 21 May 2026 15:10:51 -0400 Subject: [PATCH 63/63] revert query microservice changes --- microservices/services/query/service/pom.xml | 19 +++++-------------- .../query/QueryManagementService.java | 6 +++--- .../query/AbstractQueryServiceTest.java | 3 ++- .../query/lookup/LookupServiceTest.java | 3 ++- 4 files changed, 12 insertions(+), 19 deletions(-) diff --git a/microservices/services/query/service/pom.xml b/microservices/services/query/service/pom.xml index 149d79661c3..4ee665c5a92 100644 --- a/microservices/services/query/service/pom.xml +++ b/microservices/services/query/service/pom.xml @@ -24,15 +24,15 @@ 1.26.0 2.18.0 3.20.0 - 7.39.0-SNAPSHOT + 7.33.1 4.0.2 4.0.2 4.0.4 1.0.1 - 4.1.4-SNAPSHOT - 4.0.8-SNAPSHOT - 4.0.6-SNAPSHOT - 1.0.13-SNAPSHOT + 4.1.3 + 4.0.7 + 4.0.5 + 1.0.12 1.0.10 3.0.5 31.1-jre @@ -75,11 +75,6 @@ datawave-query-core ${version.datawave} - - gov.nsa.datawave.core - accumulo-utils - ${version.datawave} - gov.nsa.datawave.core base-rest-responses @@ -372,10 +367,6 @@ - - gov.nsa.datawave.core - accumulo-utils - gov.nsa.datawave.core base-rest-responses diff --git a/microservices/services/query/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/microservices/services/query/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 0898c12d14f..a16e1816bdd 100644 --- a/microservices/services/query/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/microservices/services/query/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -2174,7 +2174,7 @@ public void audit(String auditId, Auditor.AuditType auditType, String logicName, // if we haven't already, validate the markings SecurityMarking requestSecurityMarking = querySecurityMarking.get(); - if (requestSecurityMarking.toAccessExpressionString() == null) { + if (requestSecurityMarking.toColumnVisibilityString() == null) { validateSecurityMarkings(parameters); } @@ -2359,7 +2359,7 @@ protected Query createQuery(String queryLogicName, MultiValueMap Query query = responseObjectFactory.getQueryImpl(); query.initialize(userDn, dnList, queryLogicName, requestQueryParameters, requestQueryParameters.getUnknownParameters(parameters)); - query.setColumnVisibility(requestSecurityMarking.toAccessExpressionString()); + query.setColumnVisibility(requestSecurityMarking.toColumnVisibilityString()); query.setUncaughtExceptionHandler(new QueryUncaughtExceptionHandler()); Thread.currentThread().setUncaughtExceptionHandler(query.getUncaughtExceptionHandler()); if (queryId != null) { @@ -2602,7 +2602,7 @@ public void setInternalAuditParameters(String queryLogicName, String userDn, Mul // These are parameters that aren't passed in by the user, but rather are computed from other sources. PrivateAuditConstants.stripPrivateParameters(parameters); parameters.add(PrivateAuditConstants.LOGIC_CLASS, queryLogicName); - parameters.set(PrivateAuditConstants.COLUMN_VISIBILITY, querySecurityMarking.get().toAccessExpressionString()); + parameters.set(PrivateAuditConstants.COLUMN_VISIBILITY, querySecurityMarking.get().toColumnVisibilityString()); parameters.add(PrivateAuditConstants.USER_DN, userDn); } diff --git a/microservices/services/query/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java b/microservices/services/query/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java index 4d40c982f9d..80e9812d0bd 100644 --- a/microservices/services/query/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java +++ b/microservices/services/query/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java @@ -17,6 +17,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -167,7 +168,7 @@ protected void publishEventsToQueue(String queryId, int numEvents, MultiValueMap List fields = new ArrayList<>(); for (Map.Entry> entry : fieldValues.entrySet()) { for (String value : entry.getValue()) { - fields.add(new DefaultField(entry.getKey(), visibility, null, currentTime, value)); + fields.add(new DefaultField(entry.getKey(), visibility, new HashMap<>(), currentTime, value)); } } event.setFields(fields); diff --git a/microservices/services/query/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java b/microservices/services/query/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java index 86ec583f7b9..35d7213d22c 100644 --- a/microservices/services/query/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java +++ b/microservices/services/query/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java @@ -7,6 +7,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -876,7 +877,7 @@ protected void publishEventsToQueue(String queryId, int numEvents, MultiValueMap List fields = new ArrayList<>(); for (Map.Entry> entry : fieldValues.entrySet()) { for (String value : entry.getValue()) { - fields.add(new DefaultField(entry.getKey(), visibility, null, currentTime, value)); + fields.add(new DefaultField(entry.getKey(), visibility, new HashMap<>(), currentTime, value)); } } event.setFields(fields);