diff --git a/core/base-rest-responses/src/main/java/datawave/marking/ColumnVisibilitySecurityMarking.java b/core/base-rest-responses/src/main/java/datawave/marking/ColumnVisibilitySecurityMarking.java index 206dd12afd2..d52301ccedb 100644 --- a/core/base-rest-responses/src/main/java/datawave/marking/ColumnVisibilitySecurityMarking.java +++ b/core/base-rest-responses/src/main/java/datawave/marking/ColumnVisibilitySecurityMarking.java @@ -1,8 +1,7 @@ package datawave.marking; -import static com.google.common.base.Charsets.UTF_8; +import static datawave.marking.AccessExpressionMarkings.ACCESS; -import java.util.Collections; import java.util.List; import java.util.Map; @@ -12,7 +11,7 @@ import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlAttribute; -import org.apache.accumulo.core.security.ColumnVisibility; +import org.apache.accumulo.access.AccessExpression; import com.google.common.base.Preconditions; @@ -31,7 +30,7 @@ public void validate(Map> parameters) throws IllegalArgument if (null == values) { throw new IllegalArgumentException("Required parameter " + VISIBILITY_MARKING + " not found"); } - if (values.isEmpty() || values.size() > 1) { + if (values.size() != 1) { throw new IllegalArgumentException("Required parameter " + VISIBILITY_MARKING + " only accepts one value"); } columnVisibility = values.get(0); @@ -47,8 +46,16 @@ public void setColumnVisibility(String columnVisibility) { } @Override - public ColumnVisibility toColumnVisibility() { - return new ColumnVisibility(columnVisibility); + public AccessExpression toAccessExpression() { + if (columnVisibility == null || columnVisibility.isEmpty()) { + return ACCESS.newExpression(""); + } + return ACCESS.newExpression(columnVisibility); + } + + @Override + public Markings toMarkings() { + return AccessExpressionMarkings.builder().accessExpression(toAccessExpression()).build(); } @Override @@ -72,65 +79,29 @@ public boolean equals(Object obj) { if (this.columnVisibility == null && other.columnVisibility != null) { return false; } - if (this.columnVisibility == null && other.columnVisibility == null) { - return true; - } - if (this.columnVisibility.equals(other.columnVisibility)) { + if (this.columnVisibility == null) { return true; - } else { - return false; } + return this.columnVisibility.equals(other.columnVisibility); } return false; } @Override public String toString() { - return toColumnVisibilityString(); + return toAccessExpressionString(); } @Override - public String toColumnVisibilityString() { + public String toAccessExpressionString() { if (null == this.columnVisibility) { return null; } - return new String(toColumnVisibility().flatten(), UTF_8); + return AccessExpressionUtil.normalize(columnVisibility).getExpression(); } public void clear() { this.columnVisibility = null; } - @Override - public Map toMap() { - return Collections.singletonMap(VISIBILITY_MARKING, this.columnVisibility); - } - - @Override - public void fromMap(Map map) { - this.columnVisibility = map.get(VISIBILITY_MARKING); - } - - /** - * Turn this set of markings into a serializable string - * - * @return String - */ - @Override - public String mapToString() { - return MarkingFunctions.Encoding.toString(toMap()); - } - - /** - * Fill this security markings given an encoded string - * - * @param encodedMarkings - */ - @Override - public void fromString(String encodedMarkings) { - Map markings = MarkingFunctions.Encoding.fromString(encodedMarkings); - this.columnVisibility = markings.get(VISIBILITY_MARKING); - Preconditions.checkNotNull(columnVisibility); - } - } diff --git a/core/base-rest-responses/src/main/java/datawave/marking/SecurityMarking.java b/core/base-rest-responses/src/main/java/datawave/marking/SecurityMarking.java index d62ebd4fd1e..bb7919559fc 100644 --- a/core/base-rest-responses/src/main/java/datawave/marking/SecurityMarking.java +++ b/core/base-rest-responses/src/main/java/datawave/marking/SecurityMarking.java @@ -1,24 +1,16 @@ package datawave.marking; -import java.util.Map; - -import org.apache.accumulo.core.security.ColumnVisibility; +import org.apache.accumulo.access.AccessExpression; import datawave.validation.ParameterValidator; public interface SecurityMarking extends ParameterValidator { - ColumnVisibility toColumnVisibility() throws MarkingFunctions.Exception; - - String toColumnVisibilityString(); - - Map toMap(); - - void fromMap(Map map); + AccessExpression toAccessExpression() throws MarkingFunctions.Exception; - String mapToString(); + String toAccessExpressionString(); - void fromString(String xmlString); + Markings toMarkings() throws MarkingFunctions.Exception; void clear(); } diff --git a/core/base-rest-responses/src/main/java/datawave/webservice/query/result/event/HasMarkings.java b/core/base-rest-responses/src/main/java/datawave/webservice/query/result/event/HasMarkings.java index efc95de29a6..bfec4bcb1a7 100644 --- a/core/base-rest-responses/src/main/java/datawave/webservice/query/result/event/HasMarkings.java +++ b/core/base-rest-responses/src/main/java/datawave/webservice/query/result/event/HasMarkings.java @@ -1,10 +1,10 @@ package datawave.webservice.query.result.event; -import java.util.Map; +import datawave.marking.Markings; public interface HasMarkings { - void setMarkings(Map markings); + void setMarkings(Markings markings); - Map getMarkings(); + Markings getMarkings(); } diff --git a/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java b/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java index d50078d9b73..8ebc959801a 100644 --- a/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java +++ b/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java @@ -7,26 +7,24 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.TreeMap; import java.util.TreeSet; -import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.collect.Maps; import com.google.common.collect.Sets; import datawave.data.type.Type; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.webservice.query.cachedresults.CacheableQueryRow; import datawave.webservice.query.data.ObjectSizeOf; import datawave.webservice.query.util.TypedValue; public class CacheableQueryRowImpl extends CacheableQueryRow implements ObjectSizeOf { - private static Logger log = LoggerFactory.getLogger(CacheableQueryRowImpl.class); + private static final Logger LOGGER = LoggerFactory.getLogger(CacheableQueryRowImpl.class); private String user = null; private String queryId = null; @@ -35,77 +33,52 @@ public class CacheableQueryRowImpl extends CacheableQueryRow implements ObjectSi private String eventId = null; private String row = null; private String colFam = null; - private Map markings = new HashMap<>(); - private Map> columnMarkingsMap = new HashMap<>(); + private Markings markings = null; + private Map> columnMarkingsMap = new HashMap<>(); private Map columnColumnVisibilityMap = new HashMap<>(); - private Map columnTypeMap = Maps.newHashMap(); + private Map columnTypeMap = new HashMap<>(); private Map columnTimestampMap = new HashMap<>(); private Map> columnValues = new HashMap<>(); private Set variableColumnNames = new TreeSet<>(); private String queryOrigin = null; private String resultOrigin = null; - public void addColumn(String columnName, String columnValueString, Map markings, String columnVisibility, Long timestamp) { + public void addColumn(String columnName, String columnValueString, Markings markings, String columnVisibility, Long timestamp) { addColumn(columnName, new TypedValue(columnValueString), markings, columnVisibility, timestamp); } - public void addColumn(String columnName, TypedValue columnTypedValue, Map markings, String columnVisibility, Long timestamp) { + public void addColumn(String columnName, TypedValue columnTypedValue, Markings markings, String columnVisibility, Long timestamp) { - columnName = columnName.replaceAll(" ", "_"); + final String cleanColumnName = columnName.replace(' ', '_'); - // if new markings are the same as the old markings, skip all of this - // they are the same and the markings value has already been validated - if (this.markings.equals(markings) == false) { - if (this.markings.isEmpty()) { - // validate the markings - try { - markingFunctions.translateToColumnVisibility(markings); - if (this.markings.isEmpty()) { - // markings were empty, so use the one passed in. - this.markings = markings; - } - } catch (MarkingFunctions.Exception e) { - log.error("Invalid markings {} skipping column {} = {}", markings, columnName, columnTypedValue, e); - return; - } - } else { - try { - Set columnVisibilities = Sets.newHashSet(); - columnVisibilities.add(markingFunctions.translateToColumnVisibility(this.markings)); - columnVisibilities.add(markingFunctions.translateToColumnVisibility(markings)); - ColumnVisibility combinedVisibility = markingFunctions.combine(columnVisibilities); - - // use combined marking as new markings - this.markings = markingFunctions.translateFromColumnVisibility(combinedVisibility); - } catch (MarkingFunctions.Exception e) { - log.error("Invalid markings {} skipping column {} = {}", markings, columnName, columnTypedValue, e); - return; - } - } + try { + this.markings = markingFunctions.combine(this.markings, markings); + } catch (MarkingFunctions.Exception e) { + LOGGER.error("Invalid markings {} skipping column {} = {}", markings, cleanColumnName, columnTypedValue, e); + return; } - Long currTimestamp = columnTimestampMap.get(columnName); + Long currTimestamp = columnTimestampMap.get(cleanColumnName); if (currTimestamp == null || timestamp > currTimestamp) { - columnTimestampMap.put(columnName, timestamp); + columnTimestampMap.put(cleanColumnName, timestamp); } Type datawaveType = null; String typedColumnName = ""; if (columnTypedValue.getValue() instanceof Type) { datawaveType = (Type) columnTypedValue.getValue(); - typedColumnName = columnTypedValue.getType().replaceAll(":", "_").toUpperCase(); - typedColumnName += "_" + columnName; + typedColumnName = columnTypedValue.getType().replace(':', '_').toUpperCase(); + typedColumnName += "_" + cleanColumnName; } - manageColumnInsert(datawaveType, columnName, columnTypedValue, markings, columnVisibility); - if (!typedColumnName.isEmpty() && typedColumnName.startsWith("XS_STRING") == false) { + manageColumnInsert(datawaveType, cleanColumnName, columnTypedValue, markings, columnVisibility); + if (!typedColumnName.isEmpty() && !typedColumnName.startsWith("XS_STRING")) { manageColumnInsert(datawaveType, typedColumnName, columnTypedValue, markings, columnVisibility); } } - private void manageColumnInsert(Type datawaveType, String columnName, TypedValue columnTypedValue, Map markings, - String columnVisibility) { - if (this.columnValues.containsKey(columnName) == false) { + private void manageColumnInsert(Type datawaveType, String columnName, TypedValue columnTypedValue, Markings markings, String columnVisibility) { + if (!this.columnValues.containsKey(columnName)) { Set valuesSet = Sets.newLinkedHashSet(); if (datawaveType != null) { @@ -121,8 +94,8 @@ private void manageColumnInsert(Type datawaveType, String columnName, TypedVa this.columnValues.put(columnName, valuesSet); this.columnMarkingsMap.put(columnName, markings); this.columnColumnVisibilityMap.put(columnName, columnVisibility); - } else { + } else { if (datawaveType != null) { if (columnName.startsWith("XS")) { columnValues.get(columnName).add(datawaveType.getNormalizedValue()); @@ -133,18 +106,18 @@ private void manageColumnInsert(Type datawaveType, String columnName, TypedVa } else { columnValues.get(columnName).add(columnTypedValue.getValue().toString()); } - Map currMarkings = columnMarkingsMap.get(columnName); + Markings currMarkings = columnMarkingsMap.get(columnName); - if (currMarkings.equals(markings) == false) { + if (currMarkings == null) { + // existing marking was null so we update with new one + columnMarkingsMap.put(columnName, markings); + columnColumnVisibilityMap.put(columnName, columnVisibility); + } else if (!currMarkings.equals(markings)) { try { - Set columnVisibilities = Sets.newHashSet(); - columnVisibilities.add(markingFunctions.translateToColumnVisibility(currMarkings)); - columnVisibilities.add(markingFunctions.translateToColumnVisibility(markings)); - ColumnVisibility combinedVisibility = markingFunctions.combine(columnVisibilities); - Map minMarkings = markingFunctions.translateFromColumnVisibility(combinedVisibility); + Markings combinedMarkings = markingFunctions.combine(currMarkings, markings); // use combined marking as new markings - columnMarkingsMap.put(columnName, minMarkings); + columnMarkingsMap.put(columnName, combinedMarkings); // new combined marking means that this colVis is greater than old colVis columnColumnVisibilityMap.put(columnName, columnVisibility); } catch (MarkingFunctions.Exception e) { @@ -183,21 +156,19 @@ public String getColFam() { return colFam; } - public Map getMarkings() { - return Maps.newHashMap(markings); + public Markings getMarkings() { + return markings; } - public Map getColumnMarkings(String columnName) { - columnName = columnName.replaceAll(" ", "_"); - Map markings = null; - if (this.columnMarkingsMap.containsKey(columnName)) { - markings = this.columnMarkingsMap.get(columnName); + public Markings getColumnMarkings(String columnName) { + final String cleanColumnName = columnName.replace(' ', '_'); + Markings markings; + if (this.columnMarkingsMap.containsKey(cleanColumnName)) { + markings = this.columnMarkingsMap.get(cleanColumnName); } else { markings = this.columnMarkingsMap.get("_DEFAULT_"); } - if (markings == null) - markings = Maps.newHashMap(); return markings; } @@ -209,11 +180,7 @@ public String getColumnTimestampString(Map columnMap) { Integer currColumnNumber = columnMap.get(currColumnName); if (currColumnNumber != null) { Long currLong = entry.getValue(); - Set columnSet = timestampMap.get(currLong); - if (columnSet == null) { - columnSet = new TreeSet<>(); - timestampMap.put(currLong, columnSet); - } + Set columnSet = timestampMap.computeIfAbsent(currLong, k -> new TreeSet<>()); columnSet.add(currColumnName); } } @@ -243,10 +210,10 @@ public String getColumnTimestampString(Map columnMap) { } public String getColumnVisibility(String columnName) { - columnName = columnName.replaceAll(" ", "_"); - String columnVisibility = null; - if (this.columnColumnVisibilityMap.containsKey(columnName)) { - columnVisibility = this.columnColumnVisibilityMap.get(columnName); + final String cleanColumnName = columnName.replace(' ', '_'); + String columnVisibility; + if (this.columnColumnVisibilityMap.containsKey(cleanColumnName)) { + columnVisibility = this.columnColumnVisibilityMap.get(cleanColumnName); } else { columnVisibility = this.columnColumnVisibilityMap.get("_DEFAULT_"); } @@ -255,10 +222,10 @@ public String getColumnVisibility(String columnName) { public Long getColumnTimestamp(String columnName) { - columnName = columnName.replaceAll(" ", "_"); + final String cleanColumnName = columnName.replace(' ', '_'); Long timestamp = null; - if (this.columnTimestampMap.containsKey(columnName)) { - timestamp = this.columnTimestampMap.get(columnName); + if (this.columnTimestampMap.containsKey(cleanColumnName)) { + timestamp = this.columnTimestampMap.get(cleanColumnName); } else { timestamp = this.columnTimestampMap.get("_DEFAULT_"); } @@ -289,17 +256,11 @@ public void setColFam(String colFam) { this.colFam = colFam; } - public void setMarkings(Map markings) { - // validate the markings - try { - markingFunctions.translateToColumnVisibility(markings); - this.markings = markings; - } catch (MarkingFunctions.Exception e) { - log.error("Invalid markings {}", markings, e); - } + public void setMarkings(Markings markings) { + this.markings = markings; } - public void setColumnMarkingsMap(Map> columnMarkingsMap) { + public void setColumnMarkingsMap(Map> columnMarkingsMap) { this.columnMarkingsMap = columnMarkingsMap; } @@ -317,14 +278,12 @@ public String getColumnSecurityMarkingString(Map columnMap) { for (String field : columnMarkingsMap.keySet()) { - // sort the map - Map m = columnMarkingsMap.get(field); + Markings m = columnMarkingsMap.get(field); String v = columnColumnVisibilityMap.get(field); - if (m == null) { - m = new HashMap<>(); + String mStr = ""; + if (m != null && !m.isEmpty()) { + mStr = m.toAccessExpression().getExpression(); } - // use TreeMap to sort the map - String mStr = MarkingFunctions.Encoding.toString(new TreeMap<>(m)); if (v == null) { combinedMap.put(field, mStr); } else { @@ -338,11 +297,7 @@ public String getColumnSecurityMarkingString(Map columnMap) { Map> markingToField = new HashMap<>(); for (Map.Entry entry : combinedMap.entrySet()) { String combinedMarking = entry.getValue(); - Set fields = markingToField.get(combinedMarking); - if (fields == null) { - fields = new HashSet<>(); - markingToField.put(combinedMarking, fields); - } + Set fields = markingToField.computeIfAbsent(combinedMarking, k -> new HashSet<>()); fields.add(entry.getKey()); if (fields.size() > largestSetCount) { largestSetCount = fields.size(); @@ -377,9 +332,7 @@ public void setColumnValues(Map> columnValues) { } public List getVariableColumnNames() { - List l = new ArrayList<>(); - l.addAll(variableColumnNames); - return l; + return new ArrayList<>(variableColumnNames); } public void setVariableColumnNames(Collection c) { diff --git a/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowReader.java b/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowReader.java index 94cd75e59a8..7e32ef467ff 100644 --- a/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowReader.java +++ b/core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowReader.java @@ -9,10 +9,12 @@ import java.util.Set; import java.util.TreeSet; +import org.apache.accumulo.core.security.ColumnVisibility; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.webservice.query.cachedresults.CacheableQueryRow; import datawave.webservice.query.result.event.ResponseObjectFactory; @@ -20,8 +22,9 @@ public class CacheableQueryRowReader { private static Logger log = LoggerFactory.getLogger(CacheableQueryRowReader.class); + @SuppressWarnings("unchecked") public static CacheableQueryRow createRow(ResultSet cachedRowSet, Set fixedFieldsInEvent, ResponseObjectFactory responseObjectFactory, - MarkingFunctions markingFunctions) { + MarkingFunctions markingFunctions) { CacheableQueryRow cqfc = responseObjectFactory.getCacheableQueryRow(); cqfc.setMarkingFunctions(markingFunctions); @@ -86,22 +89,25 @@ public static CacheableQueryRow createRow(ResultSet cachedRowSet, Set fi } if (columnToIndexMap.get("_markings_") != null) { String mStr = cachedRowSet.getString(columnToIndexMap.get("_markings_")); - cqfc.setMarkings(MarkingFunctions.Encoding.fromString(mStr)); + if (mStr != null && !mStr.isEmpty()) { + cqfc.setMarkings(markingFunctions.translateFromColumnVisibility(new ColumnVisibility(mStr))); + } } if (columnToIndexMap.get("_column_markings_") != null) { String columnMarkings = cachedRowSet.getString(columnToIndexMap.get("_column_markings_")); Map combinedColumnMarkings = parseColumnMarkings(columnMarkings, columnToIndexMap); - Map> columnMarkingsMap = new HashMap<>(); + Map> columnMarkingsMap = new HashMap<>(); Map columnVisibilityMap = new HashMap<>(); for (Map.Entry entry : combinedColumnMarkings.entrySet()) { String columnName = entry.getKey(); String combinedString = entry.getValue(); int x = combinedString.lastIndexOf(':'); if (x >= 0) { - columnMarkingsMap.put(columnName, MarkingFunctions.Encoding.fromString(combinedString.substring(0, x))); + String markingStr = combinedString.substring(0, x); + columnMarkingsMap.put(columnName, markingFunctions.translateFromColumnVisibility(new ColumnVisibility(markingStr))); columnVisibilityMap.put(columnName, combinedString.substring(x + 1)); } else { - columnMarkingsMap.put(columnName, MarkingFunctions.Encoding.fromString(combinedString)); + columnMarkingsMap.put(columnName, markingFunctions.translateFromColumnVisibility(new ColumnVisibility(combinedString))); columnVisibilityMap.put(columnName, ""); } } @@ -115,6 +121,8 @@ public static CacheableQueryRow createRow(ResultSet cachedRowSet, Set fi } catch (SQLException e) { log.error(e.getMessage(), e); + } catch (MarkingFunctions.Exception e) { + throw new RuntimeException(e); } return cqfc; diff --git a/core/common-util/pom.xml b/core/common-util/pom.xml index 4024a08b2f7..896094f4d59 100644 --- a/core/common-util/pom.xml +++ b/core/common-util/pom.xml @@ -13,10 +13,6 @@ - - gov.nsa.datawave.core - accumulo-utils - gov.nsa.datawave.core datawave-core-connection-pool diff --git a/core/common-util/src/main/java/datawave/core/cache/ClassCache.java b/core/common-util/src/main/java/datawave/core/cache/ClassCache.java index 6f6238a6c96..6c0b8665b6e 100644 --- a/core/common-util/src/main/java/datawave/core/cache/ClassCache.java +++ b/core/common-util/src/main/java/datawave/core/cache/ClassCache.java @@ -27,6 +27,9 @@ public interface ClassCache { * @param name * the class name * @return a {@link Class} + * + * @throws ClassNotFoundException + * if class not found */ Class get(String name) throws ClassNotFoundException; } diff --git a/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java b/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java index c4af31664a1..584622e6d6c 100644 --- a/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java +++ b/core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java @@ -50,7 +50,9 @@ import datawave.data.type.Type; import datawave.ingest.protobuf.Uid; import datawave.ingest.protobuf.Uid.List.Builder; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.microservice.query.DefaultQueryParameters; import datawave.microservice.query.QueryPersistence; import datawave.modification.configuration.ModificationServiceConfiguration; @@ -204,7 +206,7 @@ public class MutableMetadataHandler extends ModificationServiceConfiguration { protected String reverseIndexTableName = null; protected String metadataTableName = null; protected MetadataHelperFactory metadataHelperFactory; - protected MarkingFunctions markingFunctions = null; + protected MarkingFunctions markingFunctions = null; // a map of event fields to index only/derived fields to enable appropriate deleting of event fields and all derivatives protected Multimap indexOnlyMap = null; @@ -247,11 +249,11 @@ public void setMetadataTableName(String metadataTableName) { this.metadataTableName = metadataTableName; } - public MarkingFunctions getMarkingFunctions() { + public MarkingFunctions getMarkingFunctions() { return markingFunctions; } - public void setMarkingFunctions(MarkingFunctions markingFunctions) { + public void setMarkingFunctions(MarkingFunctions markingFunctions) { this.markingFunctions = markingFunctions; } @@ -310,6 +312,24 @@ public void process(AccumuloClient client, ModificationRequestBase request, Map< this.process(client, request, mutableFieldList, userAuths, userDetails, false, true); } + /** + * Convert legacy Map<String,String> field markings (from DefaultModificationRequest) to Markings<?>. + */ + private Markings convertFieldMarkings(Map rawMarkings) { + if (rawMarkings == null || rawMarkings.isEmpty()) { + return null; + } + String vis = rawMarkings.getOrDefault("columnVisibility", ""); + if (vis.isEmpty()) { + return null; + } + try { + return markingFunctions.translateFromColumnVisibility(new ColumnVisibility(vis)); + } catch (MarkingFunctions.Exception e) { + throw new RuntimeException(e); + } + } + public void process(AccumuloClient client, ModificationRequestBase request, Map> mutableFieldList, Set userAuths, ProxiedUserDetails userDetails, boolean purgeIndex, boolean insertHistory) throws Exception { @@ -334,7 +354,7 @@ public void process(AccumuloClient client, ModificationRequestBase request, Map< String eventUid = e.getEventUid(); String oldFieldValue = null; - Map oldFieldMarkings = null; + Markings oldFieldMarkings = null; String oldColumnVisibility = null; List> currentEntryList = null; int valHistoryCount = 0; @@ -358,8 +378,7 @@ public void process(AccumuloClient client, ModificationRequestBase request, Map< // Count the history entries if history is going to be inserted. if (insertHistory && (MODE.INSERT.equals(mode) || MODE.UPDATE.equals(mode))) { - List> fieldHistoryList = getField(client, userAuths, shardId, datatype, eventUid, "HISTORY_" + fieldName, null, - new HashMap<>(), null); + List> fieldHistoryList = getField(client, userAuths, shardId, datatype, eventUid, "HISTORY_" + fieldName, null, null, null); for (Pair p : fieldHistoryList) { if (p.getLeft().getColumnQualifier().find(mr.getFieldValue()) > -1) { @@ -371,13 +390,13 @@ public void process(AccumuloClient client, ModificationRequestBase request, Map< if (MODE.UPDATE.equals(mode) || MODE.DELETE.equals(mode)) { if (MODE.UPDATE.equals(mode)) { oldFieldValue = mr.getOldFieldValue(); - oldFieldMarkings = mr.getOldFieldMarkings(); + oldFieldMarkings = convertFieldMarkings(mr.getOldFieldMarkings()); oldColumnVisibility = mr.getOldColumnVisibility(); if (null == oldFieldValue) throw new IllegalArgumentException("fieldValue parameter required for update"); } else { oldFieldValue = mr.getFieldValue(); - oldFieldMarkings = mr.getFieldMarkings(); + oldFieldMarkings = convertFieldMarkings(mr.getFieldMarkings()); oldColumnVisibility = mr.getColumnVisibility(); if (null == oldFieldValue) throw new IllegalArgumentException("fieldValue parameter required for delete"); @@ -399,7 +418,7 @@ public void process(AccumuloClient client, ModificationRequestBase request, Map< if (MODE.INSERT.equals(mode)) { String fieldValue = mr.getFieldValue(); - Map fieldMarkings = mr.getFieldMarkings(); + Markings fieldMarkings = convertFieldMarkings(mr.getFieldMarkings()); String columnVisibility = mr.getColumnVisibility(); ColumnVisibility colviz = null; if (null != columnVisibility) { @@ -414,7 +433,7 @@ public void process(AccumuloClient client, ModificationRequestBase request, Map< delete(writer, client, userAuths, currentEntryList, isIndexOnly, isIndexed, isReverseIndexed, isContent, dataTypes, user, MODE.UPDATE, origTimestamp + valHistoryCount, purgeIndex, insertHistory); String fieldValue = mr.getFieldValue(); - Map fieldMarkings = mr.getFieldMarkings(); + Markings fieldMarkings = convertFieldMarkings(mr.getFieldMarkings()); String columnVisibility = mr.getColumnVisibility(); ColumnVisibility colviz = null; if (null != columnVisibility) { @@ -636,7 +655,7 @@ protected void insertHistory(MultiTableBatchWriter writer, String shardId, Strin * @throws Exception * if there are issues */ - protected void insert(MultiTableBatchWriter writer, String shardId, String datatype, String eventUid, Map markings, ColumnVisibility viz, + protected void insert(MultiTableBatchWriter writer, String shardId, String datatype, String eventUid, Markings markings, ColumnVisibility viz, String fieldName, String fieldValue, boolean isIndexOnlyField, boolean isIndexed, boolean isReverseIndexed, Set> dataTypes, String user, MODE mode, long ts, boolean insertHistory) throws Exception { @@ -644,7 +663,7 @@ protected void insert(MultiTableBatchWriter writer, String shardId, String datat if (null == markings || markings.isEmpty()) throw new IllegalArgumentException("No security information specified. Security markings must be supplied"); - viz = markingFunctions.translateToColumnVisibility(markings); + viz = markings.toColumnVisibility(); } insert(writer, shardId, datatype, eventUid, viz, fieldName, fieldValue, ts, isIndexOnlyField, isIndexed, isReverseIndexed, dataTypes, false, @@ -816,7 +835,7 @@ protected void delete(MultiTableBatchWriter writer, AccumuloClient client, Set> getField(AccumuloClient client, Set userAuths, String shardId, String datatype, String eventUid, - String fieldName, String oldFieldValue, Map oldFieldMarkings, ColumnVisibility oldColumnVisibility) throws Exception { + String fieldName, String oldFieldValue, Markings oldFieldMarkings, ColumnVisibility oldColumnVisibility) throws Exception { Text family = new Text(datatype); TextUtil.textAppend(family, eventUid); @@ -847,19 +866,28 @@ protected List> getField(AccumuloClient client, Set markings = markingFunctions.translateFromColumnVisibilityForAuths(e.getKey().getColumnVisibilityParsed(), userAuths); - if (null != oldFieldMarkings && !oldFieldMarkings.equals(markings)) { - log.trace("Skipping key that does not match with markings: {}", e.getKey()); - continue; + try { + Markings markings = markingFunctions.translateFromColumnVisibilityForAuths(e.getKey().getColumnVisibilityParsed(), userAuths); + if (null != oldFieldMarkings && !oldFieldMarkings.equals(markings)) { + if (log.isTraceEnabled()) { + log.trace("Skipping key that does not match with markings: {}", e.getKey()); + } + continue; + } + } catch (MarkingFunctions.Exception ex) { + throw new RuntimeException(ex); } + } results.add(Pair.of(e.getKey(), e.getValue())); } diff --git a/core/modification/src/test/java/datawave/modification/MutableMetadataHandlerTestSupport.java b/core/modification/src/test/java/datawave/modification/MutableMetadataHandlerTestSupport.java index 298954850c9..916c8564f98 100644 --- a/core/modification/src/test/java/datawave/modification/MutableMetadataHandlerTestSupport.java +++ b/core/modification/src/test/java/datawave/modification/MutableMetadataHandlerTestSupport.java @@ -126,7 +126,7 @@ DefaultModificationRequest newMarkingsInsertRequest(String fieldName, String fie request.setFieldName(fieldName); request.setFieldValue(fieldValue); Map markings = new HashMap<>(); - markings.put(MarkingFunctions.Default.COLUMN_VISIBILITY, columnVisibility); + markings.put("columnVisibility", columnVisibility); request.setFieldMarkings(markings); request.setColumnVisibility(null); return request; diff --git a/core/query/src/main/java/datawave/core/query/logic/BaseQueryLogic.java b/core/query/src/main/java/datawave/core/query/logic/BaseQueryLogic.java index d52daead974..d7b90715228 100644 --- a/core/query/src/main/java/datawave/core/query/logic/BaseQueryLogic.java +++ b/core/query/src/main/java/datawave/core/query/logic/BaseQueryLogic.java @@ -47,7 +47,7 @@ public abstract class BaseQueryLogic implements QueryLogic { protected ProxiedUserDetails serverUser; protected Set requiredRoles; - protected MarkingFunctions markingFunctions; + protected MarkingFunctions markingFunctions; protected ResponseObjectFactory responseObjectFactory; protected SelectorExtractor selectorExtractor; protected ResponseEnricherBuilder responseEnricherBuilder = null; @@ -100,11 +100,11 @@ public String getPlan(AccumuloClient client, Query settings, Set return settings.getQuery(); } - public MarkingFunctions getMarkingFunctions() { + public MarkingFunctions getMarkingFunctions() { return markingFunctions; } - public void setMarkingFunctions(MarkingFunctions markingFunctions) { + public void setMarkingFunctions(MarkingFunctions markingFunctions) { this.markingFunctions = markingFunctions; } diff --git a/core/query/src/main/java/datawave/core/query/logic/BaseQueryLogicTransformer.java b/core/query/src/main/java/datawave/core/query/logic/BaseQueryLogicTransformer.java index d7928802e41..43e2935394a 100644 --- a/core/query/src/main/java/datawave/core/query/logic/BaseQueryLogicTransformer.java +++ b/core/query/src/main/java/datawave/core/query/logic/BaseQueryLogicTransformer.java @@ -4,9 +4,9 @@ public abstract class BaseQueryLogicTransformer extends AbstractQueryLogicTransformer implements QueryLogicTransformer { - protected MarkingFunctions markingFunctions; + protected MarkingFunctions markingFunctions; - public BaseQueryLogicTransformer(MarkingFunctions markingFunctions) { + public BaseQueryLogicTransformer(MarkingFunctions markingFunctions) { if (null == markingFunctions) { throw new IllegalArgumentException("MarkingFunctions must be set"); } diff --git a/core/query/src/main/java/datawave/core/query/logic/DelegatingQueryLogic.java b/core/query/src/main/java/datawave/core/query/logic/DelegatingQueryLogic.java index 40a61167d9c..d28f60842fc 100644 --- a/core/query/src/main/java/datawave/core/query/logic/DelegatingQueryLogic.java +++ b/core/query/src/main/java/datawave/core/query/logic/DelegatingQueryLogic.java @@ -251,12 +251,12 @@ public String getConnPoolName() { } @Override - public MarkingFunctions getMarkingFunctions() { + public MarkingFunctions getMarkingFunctions() { return delegate.getMarkingFunctions(); } @Override - public void setMarkingFunctions(MarkingFunctions markingFunctions) { + public void setMarkingFunctions(MarkingFunctions markingFunctions) { delegate.setMarkingFunctions(markingFunctions); } diff --git a/core/query/src/main/java/datawave/core/query/logic/QueryLogic.java b/core/query/src/main/java/datawave/core/query/logic/QueryLogic.java index 0254faf2a54..929ffc04d6a 100644 --- a/core/query/src/main/java/datawave/core/query/logic/QueryLogic.java +++ b/core/query/src/main/java/datawave/core/query/logic/QueryLogic.java @@ -317,9 +317,9 @@ default String getResponseClass(Query query) throws QueryException { Set getRequiredRoles(); - MarkingFunctions getMarkingFunctions(); + MarkingFunctions getMarkingFunctions(); - void setMarkingFunctions(MarkingFunctions markingFunctions); + void setMarkingFunctions(MarkingFunctions markingFunctions); ResponseObjectFactory getResponseObjectFactory(); diff --git a/core/utils/accumulo-utils/pom.xml b/core/utils/accumulo-utils/pom.xml index 6c21fc04660..7658125c033 100644 --- a/core/utils/accumulo-utils/pom.xml +++ b/core/utils/accumulo-utils/pom.xml @@ -9,6 +9,7 @@ accumulo-utils ${project.artifactId} + http://webservice.datawave.nsa/v1 4.1.2 1.1.7 @@ -62,15 +63,14 @@ + + io.protostuff + protostuff-api + org.apache.accumulo - accumulo-core - - - org.slf4j - slf4j-api - - + accumulo-access-core + 1.0.0-beta3 org.apache.commons @@ -80,6 +80,10 @@ org.apache.zookeeper zookeeper + + org.projectlombok + lombok + org.slf4j slf4j-api @@ -98,11 +102,22 @@ spring-jcl runtime + + com.sun.xml.bind + jaxb-impl + ${version.jaxb-impl} + test + gov.nsa.datawave datawave-in-memory-accumulo test + + io.protostuff + protostuff-core + test + org.slf4j slf4j-simple @@ -121,4 +136,56 @@ https://maven.pkg.github.com/NationalSecurityAgency/datawave + + + + true + src/main/resources + + source-templates/** + + + + + + maven-resources-plugin + + + copy-templated-sources + + copy-resources + + validate + + ${project.build.directory}/generated-sources/templated-sources + + + src/main/resources/source-templates + true + + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-source + + add-source + + generate-sources + + + target/generated-sources/templated-sources + + + + + + + diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java new file mode 100644 index 00000000000..ad902f417b6 --- /dev/null +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java @@ -0,0 +1,185 @@ +package datawave.marking; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlTransient; + +import org.apache.accumulo.access.Access; +import org.apache.accumulo.access.AccessExpression; +import org.apache.accumulo.core.security.ColumnVisibility; +import org.apache.hadoop.io.Text; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import io.protostuff.Input; +import io.protostuff.Output; +import io.protostuff.Schema; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +@XmlAccessorType(XmlAccessType.NONE) +@JsonAutoDetect(getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE) +public class AccessExpressionMarkings implements Markings { + public static final Access ACCESS = Access.builder().build(); + + public static final String COLUMN_VISIBILITY_KEY = "columnVisibility"; + + private AccessExpression accessExpression; + + @Override + public AccessExpression getMarkings() { + return getAccessExpression(); + } + + @Override + public AccessExpression toAccessExpression() { + return accessExpression; + } + + @Override + public ColumnVisibility toColumnVisibility() { + return AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.normalize(accessExpression)); + } + + @Override + public boolean isEmpty() { + return accessExpression == null || accessExpression.getExpression().isEmpty(); + } + + @JsonProperty(COLUMN_VISIBILITY_KEY) + @XmlElement(name = COLUMN_VISIBILITY_KEY) + public String getColumnVisibilityString() { + return accessExpression != null ? accessExpression.getExpression() : null; + } + + // required for JAXB unmarshalling + public void setColumnVisibilityString(String ae) { + this.accessExpression = ae != null ? ACCESS.newExpression(ae) : null; + } + + @JsonCreator + public static AccessExpressionMarkings create(@JsonProperty(COLUMN_VISIBILITY_KEY) String cv) { + if (cv != null && !cv.isEmpty()) { + return AccessExpressionMarkings.builder().columnVisibility(cv).build(); + } + return null; + } + + public static AccessExpressionMarkings createMarkings(ColumnVisibility cv) { + if (null == cv) { + return null; + } + return create(new String(cv.getExpression(), StandardCharsets.UTF_8)); + } + + public static AccessExpressionMarkings createMarkings(AccessExpression ae) { + if (null == ae) { + return null; + } + return create(ae.getExpression()); + } + + public static AccessExpressionMarkings createMarkings(Text cv) { + if (null == cv) { + return null; + } + return create(cv.toString()); + } + + public static AccessExpressionMarkings createMarkings(String cv) { + if (null == cv) { + return null; + } + return create(cv); + } + + public static class AccessExpressionMarkingsBuilder { + public AccessExpressionMarkingsBuilder columnVisibility(String cv) { + this.accessExpression = cv != null ? ACCESS.newExpression(cv) : null; + return this; + } + } + + @XmlTransient + public static final Schema SCHEMA = new Schema<>() { + private final HashMap fieldMap = new HashMap<>(); + { + fieldMap.put(COLUMN_VISIBILITY_KEY, 1); + } + + @Override + public String getFieldName(int number) { + if (number == 1) { + return COLUMN_VISIBILITY_KEY; + } + return null; + } + + @Override + public int getFieldNumber(String name) { + final Integer number = fieldMap.get(name); + return number == null ? 0 : number; + } + + @Override + public boolean isInitialized(AccessExpressionMarkings message) { + return true; + } + + @Override + public AccessExpressionMarkings newMessage() { + return AccessExpressionMarkings.builder().build(); + } + + @Override + public String messageName() { + return AccessExpressionMarkings.class.getSimpleName(); + } + + @Override + public String messageFullName() { + return AccessExpressionMarkings.class.getName(); + } + + @Override + public Class typeClass() { + return AccessExpressionMarkings.class; + } + + @Override + public void mergeFrom(Input input, AccessExpressionMarkings message) throws IOException { + int number; + while ((number = input.readFieldNumber(this)) != 0) { + if (number == 1) { + String expr = input.readString(); + if (expr != null && !expr.isEmpty()) { + message.accessExpression = ACCESS.newExpression(expr); + } + } else { + input.handleUnknownField(number, this); + } + } + } + + @Override + public void writeTo(Output output, AccessExpressionMarkings message) throws IOException { + if (message.accessExpression != null) { + output.writeString(1, message.accessExpression.getExpression(), false); + } + } + + }; +} diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionUtil.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionUtil.java new file mode 100644 index 00000000000..309ad584762 --- /dev/null +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionUtil.java @@ -0,0 +1,281 @@ +package datawave.marking; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import static datawave.marking.AccessExpressionMarkings.ACCESS; +import static org.apache.accumulo.access.ParsedAccessExpression.ExpressionType.AND; +import static org.apache.accumulo.access.ParsedAccessExpression.ExpressionType.AUTHORIZATION; + +import java.util.SortedSet; +import java.util.TreeSet; + +import org.apache.accumulo.access.Access; +import org.apache.accumulo.access.AccessExpression; +import org.apache.accumulo.access.ParsedAccessExpression; +import org.apache.accumulo.core.security.ColumnVisibility; + +/** + * Utility for bridging between Accumulo's {@link ColumnVisibility} (from accumulo-core) and {@link AccessExpression} (from accumulo-access). + *

+ * Many modules still require {@link ColumnVisibility} for the native Accumulo Key/Mutation API, while the marking functions layer has migrated to + * {@link AccessExpression}. This utility provides zero-copy-where-possible conversions between the two. + */ +public final class AccessExpressionUtil { + + private static final AccessExpression EMPTY_EXPRESSION = ACCESS.newExpression(""); + private static final ColumnVisibility EMPTY_VISIBILITY = new ColumnVisibility(); + + private AccessExpressionUtil() {} + + /** + * Convert an {@link AccessExpression} to a {@link ColumnVisibility}. + * + * @param ae + * the access expression + * @return the equivalent column visibility + */ + public static ColumnVisibility toColumnVisibility(AccessExpression ae) { + if (null == ae) { + return null; + } + String expr = ae.getExpression(); + if (expr.isEmpty()) { + return EMPTY_VISIBILITY; + } + return new ColumnVisibility(expr); + } + + /** + * Convert a {@link ColumnVisibility} to an {@link AccessExpression}. + * + * @param cv + * the column visibility + * @return the equivalent access expression + */ + public static AccessExpression toAccessExpression(ColumnVisibility cv) { + if (null == cv) { + return null; + } + byte[] expr = cv.getExpression(); + + return toAccessExpression(expr); + } + + /** + * Convert a visibility String to an {@link AccessExpression}. + * + * @param visibility + * the visibility String + * @return the equivalent access expression + */ + public static AccessExpression toAccessExpression(String visibility) { + if (visibility == null || visibility.isEmpty()) { + return EMPTY_EXPRESSION; + } + return ACCESS.newExpression(visibility); + } + + /** + * Convert a raw visibility byte array (e.g. from {@code Key.getColumnVisibilityData()}) to an {@link AccessExpression}. + * + * @param visibilityBytes + * the raw visibility bytes + * @return the equivalent access expression + */ + public static AccessExpression toAccessExpression(byte[] visibilityBytes) { + if (visibilityBytes == null || visibilityBytes.length == 0) { + return EMPTY_EXPRESSION; + } + return ACCESS.newExpression(new String(visibilityBytes, UTF_8)); + } + + /** + * Normalize an {@link AccessExpression} by deduplicating and sorting terms. This is the replacement for the old {@code ColumnVisibility.flatten()} + * behavior. + * + * @param expression + * the expression to normalize + * @return a normalized {@link AccessExpression} + */ + public static AccessExpression normalize(AccessExpression expression) { + if (null == expression) { + return null; + } + return normalize(expression.getExpression()); + } + + /** + * Normalize a ColumnVisibility by deduplicating and sorting terms. This is the replacement for the old {@code ColumnVisibility.flatten()} behavior. + * + * @param cv + * the column visibility to normalize + * @return a normalized {@link AccessExpression} + */ + public static AccessExpression normalize(ColumnVisibility cv) { + if (null == cv) { + return null; + } + return normalize(new String(cv.getExpression(), UTF_8)); + } + + /** + * Normalize a ColumnVisibility String by deduplicating and sorting terms. This is the replacement for the old {@code ColumnVisibility.flatten()} behavior. + * + * @param cv + * the column visibility to normalize + * @return a normalized {@link AccessExpression} + */ + public static AccessExpression normalize(String cv) { + if (null == cv) { + return null; + } + ParsedAccessExpression parsed = ACCESS.newParsedExpression(cv); + return ACCESS.newExpression(normalize(parsed).expression); + } + + /** + * @return the shared {@link Access} instance used by this utility + */ + public static Access getAccess() { + return ACCESS; + } + + /** + * @return a cached empty {@link AccessExpression} + */ + public static AccessExpression emptyExpression() { + return EMPTY_EXPRESSION; + } + + /** + * As part of normalizing access expression this class is used to sort and dedupe sub-expressions in a tree set. + */ + public static class NormalizedExpression implements Comparable { + public final String expression; + public final ParsedAccessExpression.ExpressionType type; + + NormalizedExpression(String expression, ParsedAccessExpression.ExpressionType type) { + this.expression = expression; + this.type = type; + } + + // determines the sort order of different kinds of subexpressions. + private static int typeOrder(ParsedAccessExpression.ExpressionType type) { + switch (type) { + case AUTHORIZATION: + return 1; + case OR: + return 2; + case AND: + return 3; + case EMPTY: + default: + throw new IllegalArgumentException("Unexpected type " + type); + } + } + + @Override + public int compareTo(NormalizedExpression o) { + // Changing this comparator would significantly change how expressions are normalized. + int cmp = typeOrder(type) - typeOrder(o.type); + if (cmp == 0) { + if (type == AUTHORIZATION) { + // sort based on the unquoted and unescaped form of the authorization + cmp = ACCESS.unquote(expression).compareTo(ACCESS.unquote(o.expression)); + } else { + cmp = expression.compareTo(o.expression); + } + + } + return cmp; + } + + @Override + public boolean equals(Object o) { + return this == o || (o instanceof NormalizedExpression && compareTo((NormalizedExpression) o) == 0); + } + + @Override + public int hashCode() { + return expression.hashCode(); + } + } + + /** + * This method helps with the flattening aspect of normalization by recursing down as far as possible the parse tree in the case when the expression type is + * the same. As long as the type is the same in the sub expression, keep using the same set. + */ + private static void flatten(ParsedAccessExpression.ExpressionType parentType, ParsedAccessExpression parsed, + SortedSet normalizedExpressions) { + if (parsed.getType() == parentType) { + for (var child : parsed.getChildren()) { + flatten(parentType, child, normalizedExpressions); + } + } else { + // The type changed, so start again on the subexpression. + normalizedExpressions.add(normalize(parsed)); + } + } + + /** + *

+ * For a given access expression this example will deduplicate, sort, flatten, and remove unneeded parentheses or quotes in the expressions. The following + * list gives examples of what each normalization step does. + * + *

    + *
  • As an example of flattening, the expression {@code A&(B&C)} flattens to {@code + * A&B&C}.
    • + *
  • As an example of sorting, the expression {@code (Z&Y)|(C&B)} sorts to {@code + * (B&C)|(Y&Z)}
    • + *
  • As an example of deduplication, the expression {@code X&Y&X} normalizes to {@code X&Y}
    • + *
  • As an example of unneeded quotes, the expression {@code "ABC"&"XYZ"} normalizes to {@code ABC&XYZ}
    • + *
  • As an example of unneeded parentheses, the expression {@code (((ABC)|(XYZ)))} normalizes to {@code ABC|XYZ}
    • + *
+ * + *

+ * This algorithm attempts to have the same behavior as the one in the Accumulo 2.1 ColumnVisibility class. However the implementation is very different. + */ + public static NormalizedExpression normalize(ParsedAccessExpression parsed) { + if (parsed.getType() == AUTHORIZATION) { + // If the authorization is quoted and it does not need to be quoted then the following two + // lines will remove the unnecessary quoting. + String unquoted = ACCESS.unquote(parsed.getExpression()); + String quoted = ACCESS.quote(unquoted); + return new NormalizedExpression(quoted, parsed.getType()); + } else { + // The tree set does the work of sorting and deduplicating sub expressions. + TreeSet normalizedChildren = new TreeSet<>(); + for (var child : parsed.getChildren()) { + flatten(parsed.getType(), child, normalizedChildren); + } + + if (normalizedChildren.size() == 1) { + return normalizedChildren.first(); + } else { + StringBuilder builder = getStringBuilder(parsed, normalizedChildren); + + return new NormalizedExpression(builder.toString(), parsed.getType()); + } + } + } + + private static StringBuilder getStringBuilder(ParsedAccessExpression parsed, TreeSet normalizedChildren) { + String operator = parsed.getType() == AND ? "&" : "|"; + String sep = ""; + + StringBuilder builder = new StringBuilder(); + + for (var child : normalizedChildren) { + builder.append(sep); + if (child.type == AUTHORIZATION) { + builder.append(child.expression); + } else { + builder.append("("); + builder.append(child.expression); + builder.append(")"); + } + sep = operator; + } + return builder; + } +} diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/ColumnVisibilityHelper.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/ColumnVisibilityHelper.java deleted file mode 100644 index 7170eb64428..00000000000 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/ColumnVisibilityHelper.java +++ /dev/null @@ -1,133 +0,0 @@ -package datawave.marking; - -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import org.apache.accumulo.core.security.Authorizations; -import org.apache.accumulo.core.security.ColumnVisibility; -import org.apache.accumulo.core.security.ColumnVisibility.Node; -import org.apache.accumulo.core.security.ColumnVisibility.NodeType; -import org.apache.accumulo.core.security.VisibilityEvaluator; -import org.apache.accumulo.core.security.VisibilityParseException; -import org.apache.log4j.Logger; - -import datawave.marking.MarkingFunctions.Exception; - -public class ColumnVisibilityHelper { - - protected static final Charset charset = StandardCharsets.UTF_8; - private static Logger log = Logger.getLogger(ColumnVisibilityHelper.class); - - public static ColumnVisibility simplifyColumnVisibilityForAuthorizations(ColumnVisibility columnVisibility, Collection authorizations) - throws MarkingFunctions.Exception { - - ColumnVisibility simplifiedCV = columnVisibility; - - Node node = columnVisibility.getParseTree(); - if (node.getType() == NodeType.OR) { - if (log.isTraceEnabled()) { - log.trace("Top level OR Node, removing unsatisfied branches from: " + columnVisibility); - } - byte[] expression = columnVisibility.getExpression(); - if (authorizations != null) { - HashSet ve = new HashSet<>(); - for (Authorizations a : authorizations) - ve.add(new VisibilityEvaluator(a)); - removeUnsatisfiedTopLevelOrNodes(ve, expression, node); - } - - simplifiedCV = ColumnVisibilityHelper.flatten(node, expression); - if (log.isTraceEnabled()) { - log.trace("removed unsatisfied branches, visibility now: " + simplifiedCV); - } - } - return simplifiedCV; - } - - public static ColumnVisibility removeUndisplayedVisibilities(ColumnVisibility columnVisibility, Set undisplayedVisibilities) - throws MarkingFunctions.Exception { - ColumnVisibility newColumnVisibility = columnVisibility; - if (undisplayedVisibilities != null && !undisplayedVisibilities.isEmpty()) { - byte[] expression = columnVisibility.getExpression(); - Node node = columnVisibility.getParseTree(); - removeUndisplayedVisibilities(node, expression, undisplayedVisibilities); - newColumnVisibility = flatten(node, expression); - } - return newColumnVisibility; - } - - private static void removeUnsatisfiedTopLevelOrNodes(Set ve, byte[] expression, Node node) throws Exception { - if (node.getType() == NodeType.OR) { - List children = node.getChildren(); - int lastNode = children.size() - 1; - for (int x = lastNode; x >= 0; x--) { - Node currNode = children.get(x); - boolean remove = isUnsatisfied(x, ve, currNode, expression); - if (remove == true) { - children.remove(x); - } - } - } - } - - private static boolean isUnsatisfied(int position, Set ve, Node currNode, byte[] expression) throws Exception { - boolean unsatisfied = false; - try { - ColumnVisibility currVis = ColumnVisibilityHelper.flatten(currNode, expression); - for (VisibilityEvaluator v : ve) { - if (!v.evaluate(currVis)) - unsatisfied = true; - } - } catch (VisibilityParseException e) { - throw new MarkingFunctions.Exception(e); - } - return unsatisfied; - } - - private static String termNodeToString(Node termNode, byte[] expression) throws Exception { - int start = termNode.getTermStart(); - int end = termNode.getTermEnd(); - - String str = "[ERROR]"; - try { - str = new String(expression, start, end - start, charset); - } catch (RuntimeException e) { - log.error("Error converting term: start:" + start + " length:" + (end - start) + "of expression:" + expression + " -- " + e.getMessage()); - throw new MarkingFunctions.Exception(e); - } - return str; - } - - private static ColumnVisibility flatten(Node node, byte[] expression) { - - Node newNode = ColumnVisibility.normalize(node, expression); - StringBuilder sb = new StringBuilder(); - ColumnVisibility.stringify(newNode, expression, sb); - return new ColumnVisibility(sb.toString()); - } - - private static void removeUndisplayedVisibilities(Node node, byte[] expression, Set undisplayedVisibilities) throws MarkingFunctions.Exception { - List children = node.getChildren(); - // walk backwards so we don't change the index that we need to remove - int lastNode = children.size() - 1; - for (int x = lastNode; x >= 0; x--) { - Node currNode = children.get(x); - if (currNode.getType() == NodeType.TERM) { - String nodeString = termNodeToString(currNode, expression); - if (undisplayedVisibilities.contains(nodeString)) { - children.remove(x); - } - } else { - removeUndisplayedVisibilities(currNode, expression, undisplayedVisibilities); - // if all children of this NodeType.AND or NodeType.OR node have been removed, remove the node itself. - if (currNode.getChildren().isEmpty()) { - children.remove(x); - } - } - } - } -} diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/FlattenedVisibilityCache.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/FlattenedVisibilityCache.java index 85e1697cc26..2b77662c99b 100644 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/FlattenedVisibilityCache.java +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/FlattenedVisibilityCache.java @@ -13,7 +13,7 @@ * */ public class FlattenedVisibilityCache { - private static Map flattenedVisCache = Collections.synchronizedMap(new HashMap<>()); + private static final Map flattenedVisCache = Collections.synchronizedMap(new HashMap<>()); /** * Create a flattened visibility, using the cache if possible diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java index 9026edc78df..a6034a353a8 100644 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java @@ -1,27 +1,23 @@ package datawave.marking; -import static java.nio.charset.StandardCharsets.UTF_8; +import static datawave.marking.AccessExpressionMarkings.ACCESS; -import java.io.IOException; -import java.lang.reflect.InvocationTargetException; -import java.util.Arrays; +import java.nio.charset.StandardCharsets; import java.util.Collection; -import java.util.Collections; -import java.util.Map; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; import java.util.stream.Collectors; +import org.apache.accumulo.access.AccessExpression; +import org.apache.accumulo.access.ParsedAccessExpression; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.ColumnVisibility; -import org.apache.commons.beanutils.BeanUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.support.ClassPathXmlApplicationContext; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Charsets; -import com.google.common.collect.Maps; - /** * Accumulo marks all data with a columnVisibility that declares and controls access. MarkingFunctions provide a pattern for mapping a user's preferred means of * declaring access controls with the Accumulo columnVisibility pattern. As an example, James Bond might use MarkingFunctions to translate a the @@ -29,26 +25,23 @@ * human-readable marking like "Patient Privileged Information" */ -public interface MarkingFunctions { +public interface MarkingFunctions> { - ColumnVisibility combine(Collection columnVisibilities) throws MarkingFunctions.Exception; + ColumnVisibility combineVisibilities(Collection visibilities) throws MarkingFunctions.Exception; - @SuppressWarnings("unchecked") - Map combine(Map... markings) throws MarkingFunctions.Exception; + T combine(Collection> markings) throws MarkingFunctions.Exception; - ColumnVisibility translateToColumnVisibility(Map markings) throws MarkingFunctions.Exception; + T combine(Markings markings1, Markings markings2) throws MarkingFunctions.Exception; - Map translateFromColumnVisibility(ColumnVisibility columnVisibility) throws MarkingFunctions.Exception; + Markings translateFromColumnVisibility(ColumnVisibility columnVisibility) throws MarkingFunctions.Exception; - Map translateFromColumnVisibilityForAuths(ColumnVisibility columnVisibility, Collection authorizations) + Markings translateFromColumnVisibilityForAuths(ColumnVisibility columnVisibility, Collection authorizations) throws MarkingFunctions.Exception; - Map translateFromColumnVisibilityForAuths(ColumnVisibility columnVisibility, Authorizations authorizations) - throws MarkingFunctions.Exception; + Markings translateFromColumnVisibilityForAuths(ColumnVisibility columnVisibility, Authorizations authorizations) throws MarkingFunctions.Exception; byte[] flatten(ColumnVisibility vis); - @SuppressWarnings("serial") class Exception extends java.lang.Exception { public Exception() { @@ -72,47 +65,100 @@ public Exception(Throwable cause) { } } - class Default implements MarkingFunctions { - public static final String COLUMN_VISIBILITY = "columnVisibility"; + class Default implements MarkingFunctions { @Override - public ColumnVisibility combine(Collection expressions) { + public ColumnVisibility combineVisibilities(Collection visibilities) { + AccessExpressionMarkings accessExpressionMarkings = combine( + visibilities.stream().filter(Objects::nonNull).map(AccessExpressionMarkings::createMarkings).collect(Collectors.toList())); + + if (null == accessExpressionMarkings) { + // is this correct behavior, does it match other implementations? + // switching to null causes lots of NPEs so leaving this as is + return new ColumnVisibility(); + } - // filter out any empty expressions, then flatten each one (to de-dupe) and concatenate with '&' - // flatten the final combined ColumnVisibility and use that to make the ColumnVisibility to return - return new ColumnVisibility(new ColumnVisibility(expressions.stream().map(ColumnVisibility::flatten).filter(b -> b.length > 0) - .map(b -> "(" + new String(b, UTF_8) + ")").collect(Collectors.joining("&")).getBytes(UTF_8)).flatten()); + return accessExpressionMarkings.toColumnVisibility(); } @Override - @SafeVarargs - public final Map combine(Map... markings) { - // translate COLUMN_VISIBILITY values to ColumnVisibility, combine them and - // return translated back to Map - return translateFromColumnVisibility(combine(Arrays.stream(markings).filter(m -> m.containsKey(COLUMN_VISIBILITY)) - .map(this::translateToColumnVisibility).collect(Collectors.toSet()))); + public AccessExpressionMarkings combine(Collection> markings) { + + Set uniqueMarkings = new HashSet<>(); + + for (Markings marking : markings) { + if (null == marking) { + continue; + } + if (!(marking instanceof AccessExpressionMarkings)) { + throw new RuntimeException(String.format("Unknown markings class %s", marking.getClass().getName())); + } + uniqueMarkings.add((AccessExpressionMarkings) marking); + } + + if (uniqueMarkings.isEmpty()) { + return null; + } + + if (uniqueMarkings.size() == 1) { + return uniqueMarkings.stream().findFirst().get(); + } + + Set uniqueExpressions = uniqueMarkings.stream().map(AccessExpressionMarkings::getMarkings).collect(Collectors.toSet()); + + // filter out any empty expressions and concatenate with '&' + // @formatter:off + ParsedAccessExpression expression = ACCESS.newParsedExpression(uniqueExpressions + .stream() + .map(AccessExpression::getExpression) + .filter(str -> !str.isEmpty()) + .map(str -> "(" + str + ")") + .collect(Collectors.joining("&")) + ); + // @formatter:on + + // normalize the Parsed Expression and then return a copy without the parse tree + return AccessExpressionMarkings.builder().columnVisibility(AccessExpressionUtil.normalize(expression).expression).build(); } @Override - public ColumnVisibility translateToColumnVisibility(Map markings) { - ColumnVisibility cv = new ColumnVisibility(markings.get(COLUMN_VISIBILITY)); - return new ColumnVisibility(cv.flatten()); + public AccessExpressionMarkings combine(Markings markings1, Markings markings2) { + + if (markings1 == null && markings2 == null) { + return null; + } + + if (markings2 == null) { + return (AccessExpressionMarkings) markings1; + } + + if (markings1 == null) { + return (AccessExpressionMarkings) markings2; + } + + if (markings1 instanceof AccessExpressionMarkings && markings2 instanceof AccessExpressionMarkings) { + AccessExpressionMarkings aem1 = (AccessExpressionMarkings) markings1; + AccessExpressionMarkings aem2 = (AccessExpressionMarkings) markings2; + return combine(List.of(aem1, aem2)); + } + + throw new RuntimeException(String.format("Unknown markings class %s or %s", markings1.getClass().getName(), markings2.getClass().getName())); + } @Override - public Map translateFromColumnVisibility(ColumnVisibility expression) { - Map markings = Maps.newHashMap(); - markings.put(COLUMN_VISIBILITY, new String(expression.getExpression(), Charsets.UTF_8)); - return markings; + public Markings translateFromColumnVisibility(ColumnVisibility expression) { + String cv = new String(expression.getExpression(), StandardCharsets.UTF_8); + return AccessExpressionMarkings.builder().accessExpression(AccessExpressionUtil.normalize(cv)).build(); } @Override - public Map translateFromColumnVisibilityForAuths(ColumnVisibility columnVisibility, Collection authorizations) { + public Markings translateFromColumnVisibilityForAuths(ColumnVisibility columnVisibility, Collection authorizations) { return translateFromColumnVisibility(columnVisibility); } @Override - public Map translateFromColumnVisibilityForAuths(ColumnVisibility columnVisibility, Authorizations authorizations) { + public Markings translateFromColumnVisibilityForAuths(ColumnVisibility columnVisibility, Authorizations authorizations) { return translateFromColumnVisibility(columnVisibility); } @@ -120,58 +166,6 @@ public Map translateFromColumnVisibilityForAuths(ColumnVisibility public byte[] flatten(ColumnVisibility vis) { return FlattenedVisibilityCache.flatten(vis); } - - } - - class Util { - - public static Object populate(Object obj, Map source) { - try { - BeanUtils.populate(obj, source); - } catch (IllegalAccessException | InvocationTargetException e) { - throw new RuntimeException("Error populating object: " + obj.getClass().getName(), e); - } - return obj; - } - } - - class Encoding { - static private Logger log = LoggerFactory.getLogger(Encoding.class); - - /** - * Turn a set of markings into a serializable string - * - * @param markings - * the markings map to convert to a string - * @return a serialized String version of {@code markings} - */ - public static String toString(Map markings) { - ObjectMapper objectMapper = new ObjectMapper(); - try { - return objectMapper.writeValueAsString(markings); - } catch (JsonProcessingException e) { - log.error("could not serialize " + markings); - return ""; - } - } - - /** - * Turn a serialized set of markings into a map - * - * @param encodedMarkings - * the serialized String markings to convert back to a markings Map - * @return a {@link Map} of the de-serialized markings from {@code encodedMarkings} - */ - public static Map fromString(String encodedMarkings) { - ObjectMapper objectMapper = new ObjectMapper(); - try { - // noinspection unchecked - return objectMapper.readValue(encodedMarkings, Map.class); - } catch (IOException e) { - log.error("could not deserialize " + encodedMarkings); - return Collections.emptyMap(); - } - } } /** @@ -180,9 +174,9 @@ public static Map fromString(String encodedMarkings) { class Factory { public static final Logger log = LoggerFactory.getLogger(Factory.class); - private static MarkingFunctions markingFunctions; + private static MarkingFunctions markingFunctions; - public static synchronized MarkingFunctions createMarkingFunctions() { + public static synchronized MarkingFunctions createMarkingFunctions() { if (markingFunctions != null) return markingFunctions; ClassLoader thisClassLoader = Factory.class.getClassLoader(); diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/Markings.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/Markings.java new file mode 100644 index 00000000000..2d8a8e7d2cd --- /dev/null +++ b/core/utils/accumulo-utils/src/main/java/datawave/marking/Markings.java @@ -0,0 +1,20 @@ +package datawave.marking; + +import org.apache.accumulo.access.AccessExpression; +import org.apache.accumulo.core.security.ColumnVisibility; + +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS) +public interface Markings { + + T getMarkings(); + + AccessExpression toAccessExpression(); + + ColumnVisibility toColumnVisibility(); + + default boolean isEmpty() { + return getMarkings() == null; + } +} diff --git a/core/utils/accumulo-utils/src/main/java/datawave/marking/VisibilityFlattener.java b/core/utils/accumulo-utils/src/main/java/datawave/marking/VisibilityFlattener.java deleted file mode 100644 index 63cec96d4e4..00000000000 --- a/core/utils/accumulo-utils/src/main/java/datawave/marking/VisibilityFlattener.java +++ /dev/null @@ -1,46 +0,0 @@ -package datawave.marking; - -import static java.nio.charset.StandardCharsets.UTF_8; - -import java.util.Arrays; - -import org.apache.accumulo.core.security.ColumnVisibility; -import org.apache.accumulo.core.security.ColumnVisibility.Node; -import org.apache.accumulo.core.security.ColumnVisibility.NodeComparator; -import org.apache.accumulo.core.security.ColumnVisibility.NodeType; -import org.apache.hadoop.io.Text; - -public class VisibilityFlattener { - public static ColumnVisibility flatten(Node root, byte[] expression, boolean sort) { - StringBuilder out = new StringBuilder(); - flatten(root, expression, out, sort); - return new ColumnVisibility(out.toString()); - } - - public static Text flattenToText(Node root, byte[] expression, boolean sort) { - StringBuilder out = new StringBuilder(); - flatten(root, expression, out, sort); - return new Text(out.toString()); - } - - private static void flatten(Node root, byte[] expression, StringBuilder out, boolean sort) { - if (root.getType() == NodeType.TERM) - out.append(new String(expression, root.getTermStart(), root.getTermEnd() - root.getTermStart(), UTF_8)); - else { - String sep = ""; - Node[] children = root.getChildren().toArray(new Node[] {}); - if (sort) - Arrays.sort(children, new NodeComparator(expression)); - for (Node c : children) { - out.append(sep); - boolean parens = (c.getType() != NodeType.TERM && root.getType() != c.getType()); - if (parens) - out.append("("); - flatten(c, expression, out, sort); - if (parens) - out.append(")"); - sep = root.getType() == NodeType.AND ? "&" : "|"; - } - } - } -} diff --git a/core/utils/accumulo-utils/src/main/resources/source-templates/datawave/marking/package-info.java b/core/utils/accumulo-utils/src/main/resources/source-templates/datawave/marking/package-info.java new file mode 100644 index 00000000000..6a47ac1a52a --- /dev/null +++ b/core/utils/accumulo-utils/src/main/resources/source-templates/datawave/marking/package-info.java @@ -0,0 +1,6 @@ +@XmlSchema(namespace="${datawave.webservice.namespace}", elementFormDefault=XmlNsForm.QUALIFIED, xmlns={@XmlNs(prefix = "", namespaceURI = "${datawave.webservice.namespace}")}) +package datawave.marking; + +import javax.xml.bind.annotation.XmlNs; +import javax.xml.bind.annotation.XmlNsForm; +import javax.xml.bind.annotation.XmlSchema; diff --git a/core/utils/accumulo-utils/src/test/java/datawave/marking/AccessExpressionMarkingsTest.java b/core/utils/accumulo-utils/src/test/java/datawave/marking/AccessExpressionMarkingsTest.java new file mode 100644 index 00000000000..95401fcffe1 --- /dev/null +++ b/core/utils/accumulo-utils/src/test/java/datawave/marking/AccessExpressionMarkingsTest.java @@ -0,0 +1,385 @@ +package datawave.marking; + +import static datawave.marking.AccessExpressionMarkings.ACCESS; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBElement; +import javax.xml.bind.JAXBException; +import javax.xml.bind.Marshaller; +import javax.xml.bind.Unmarshaller; +import javax.xml.namespace.QName; +import javax.xml.transform.stream.StreamSource; + +import org.apache.accumulo.access.AccessExpression; +import org.apache.accumulo.access.InvalidAccessExpressionException; +import org.apache.accumulo.core.security.ColumnVisibility; +import org.apache.hadoop.io.Text; +import org.junit.jupiter.api.Test; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import io.protostuff.LinkedBuffer; +import io.protostuff.ProtostuffIOUtil; +import io.protostuff.Schema; + +public class AccessExpressionMarkingsTest { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final Schema SCHEMA = AccessExpressionMarkings.SCHEMA; + + // -- getMarkings / toAccessExpression -- + + @Test + public void testGetMarkingsReturnsAccessExpression() { + AccessExpression ae = ACCESS.newExpression("A&B"); + AccessExpressionMarkings markings = AccessExpressionMarkings.builder().accessExpression(ae).build(); + assertEquals(ae, markings.getMarkings()); + assertEquals(ae, markings.toAccessExpression()); + } + + // -- toColumnVisibility -- + + @Test + public void testToColumnVisibility() { + AccessExpressionMarkings markings = AccessExpressionMarkings.create("B&A"); + ColumnVisibility cv = markings.toColumnVisibility(); + // normalized: sorted + assertEquals("A&B", new String(cv.getExpression())); + } + + // -- isEmpty -- + + @Test + public void testIsEmptyWithNullExpression() { + AccessExpressionMarkings markings = AccessExpressionMarkings.builder().accessExpression(null).build(); + assertTrue(markings.isEmpty()); + } + + @Test + public void testIsEmptyWithEmptyExpression() { + assertNull(AccessExpressionMarkings.create("")); + } + + @Test + public void testIsEmptyWithNonEmptyExpression() { + AccessExpressionMarkings markings = AccessExpressionMarkings.create("A"); + assertFalse(markings.isEmpty()); + } + + // -- getColumnVisibilityString -- + + @Test + public void testGetColumnVisibilityString() { + AccessExpressionMarkings markings = AccessExpressionMarkings.create("X|Y"); + assertEquals("X|Y", markings.getColumnVisibilityString()); + } + + @Test + public void testGetColumnVisibilityStringNull() { + AccessExpressionMarkings markings = AccessExpressionMarkings.builder().accessExpression(null).build(); + assertNull(markings.getColumnVisibilityString()); + } + + // -- create (JsonCreator) -- + + @Test + public void testCreateWithNull() { + assertNull(AccessExpressionMarkings.create(null)); + } + + @Test + public void testCreateWithExpression() { + AccessExpressionMarkings markings = AccessExpressionMarkings.create("A&B&C"); + assertEquals("A&B&C", markings.getColumnVisibilityString()); + } + + @Test + public void testCreateIllegalOperator1() { + assertThrows(InvalidAccessExpressionException.class, () -> AccessExpressionMarkings.create("A&&B&C")); + } + + @Test + public void testCreateIllegalOperator2() { + assertThrows(InvalidAccessExpressionException.class, () -> AccessExpressionMarkings.create("A||B&C")); + } + + @Test + public void testCreateMalformed1() { + assertThrows(InvalidAccessExpressionException.class, () -> AccessExpressionMarkings.create("(A&B&C))")); + } + + @Test + public void testCreateMalformed2() { + assertThrows(InvalidAccessExpressionException.class, () -> AccessExpressionMarkings.create("(A&B,C))")); + } + + // -- createMarkings(ColumnVisibility) -- + + @Test + public void testCreateMarkingsFromColumnVisibilityNull() { + assertNull(AccessExpressionMarkings.createMarkings((ColumnVisibility) null)); + } + + @Test + public void testCreateMarkingsFromColumnVisibility() { + ColumnVisibility cv = new ColumnVisibility("A|B"); + AccessExpressionMarkings markings = AccessExpressionMarkings.createMarkings(cv); + assertEquals("A|B", markings.getColumnVisibilityString()); + } + + // -- createMarkings(AccessExpression) -- + + @Test + public void testCreateMarkingsFromAccessExpressionNull() { + assertNull(AccessExpressionMarkings.createMarkings((AccessExpression) null)); + } + + @Test + public void testCreateMarkingsFromAccessExpression() { + AccessExpression ae = ACCESS.newExpression("C&D"); + AccessExpressionMarkings markings = AccessExpressionMarkings.createMarkings(ae); + assertEquals("C&D", markings.getColumnVisibilityString()); + } + + // -- createMarkings(Text) -- + + @Test + public void testCreateMarkingsFromTextNull() { + assertNull(AccessExpressionMarkings.createMarkings((Text) null)); + } + + @Test + public void testCreateMarkingsFromText() { + AccessExpressionMarkings markings = AccessExpressionMarkings.createMarkings(new Text("E|F")); + assertEquals("E|F", markings.getColumnVisibilityString()); + } + + // -- createMarkings(String) -- + + @Test + public void testCreateMarkingsFromStringNull() { + assertNull(AccessExpressionMarkings.createMarkings((String) null)); + } + + @Test + public void testCreateMarkingsFromString() { + AccessExpressionMarkings markings = AccessExpressionMarkings.createMarkings("G&H"); + assertEquals("G&H", markings.getColumnVisibilityString()); + } + + // -- JSON round-trip -- + + @Test + public void testJsonSerialization() throws Exception { + AccessExpressionMarkings original = AccessExpressionMarkings.create("A&B"); + String json = MAPPER.writeValueAsString(original); + assertTrue(json.contains("\"columnVisibility\":\"A&B\"")); + } + + @Test + public void testJsonDeserialization() throws Exception { + String json = "{\"@class\":\"datawave.marking.AccessExpressionMarkings\",\"columnVisibility\":\"X|Y\"}"; + AccessExpressionMarkings markings = MAPPER.readValue(json, AccessExpressionMarkings.class); + assertEquals("X|Y", markings.getColumnVisibilityString()); + } + + @Test + public void testJsonRoundTrip() throws Exception { + AccessExpressionMarkings original = AccessExpressionMarkings.create("A&B&C"); + String json = MAPPER.writeValueAsString(original); + // round-trip through the polymorphic Markings type + Markings deserialized = MAPPER.readValue(json, Markings.class); + assertInstanceOf(AccessExpressionMarkings.class, deserialized); + assertEquals(original.getColumnVisibilityString(), ((AccessExpressionMarkings) deserialized).getColumnVisibilityString()); + } + + // -- equals via Lombok @Data -- + + @Test + public void testEquality() { + AccessExpressionMarkings a = AccessExpressionMarkings.create("A&B"); + AccessExpressionMarkings b = AccessExpressionMarkings.create("A&B"); + assertEquals(a, b); + assertEquals(a.hashCode(), b.hashCode()); + } + + // -- JAXB round-trip -- + + @Test + public void testJaxbMarshal() throws Exception { + AccessExpressionMarkings original = AccessExpressionMarkings.create("A&B"); + String xml = makeJaxbMarshall(original); + assertTrue(xml.contains("A&B"), "XML should contain the columnVisibility element: " + xml); + } + + @Test + public void testJaxbUnmarshal() throws Exception { + String expression = "X|Y"; + AccessExpressionMarkings markings = makeJaxbUnmarshall(expression); + assertEquals("X|Y", markings.getColumnVisibilityString()); + assertFalse(markings.isEmpty()); + } + + @Test + public void testJaxbRoundTrip() throws Exception { + AccessExpressionMarkings original = AccessExpressionMarkings.create("(A&B)|C"); + AccessExpressionMarkings result = makeJaxbRoundTrip(original); + assertEquals(original.getColumnVisibilityString(), result.getColumnVisibilityString()); + } + + @Test + public void testJaxbRoundTripEmpty() throws Exception { + AccessExpressionMarkings original = AccessExpressionMarkings.builder().build(); + AccessExpressionMarkings result = makeJaxbRoundTrip(original); + + assertNull(result.getColumnVisibilityString()); + assertTrue(result.isEmpty()); + } + + @Test + public void testJaxbRoundTripComplexExpression() throws Exception { + AccessExpressionMarkings original = AccessExpressionMarkings.create("(A&B)|(C&D&E)"); + AccessExpressionMarkings result = makeJaxbRoundTrip(original); + + assertEquals("(A&B)|(C&D&E)", result.getColumnVisibilityString()); + assertFalse(result.isEmpty()); + } + // -- Protostuff Schema metadata -- + + @Test + public void testSchemaGetFieldNameKnown() { + assertEquals(AccessExpressionMarkings.COLUMN_VISIBILITY_KEY, SCHEMA.getFieldName(1)); + } + + @Test + public void testSchemaGetFieldNameUnknown() { + assertNull(SCHEMA.getFieldName(0)); + assertNull(SCHEMA.getFieldName(2)); + } + + @Test + public void testSchemaGetFieldNumberKnown() { + assertEquals(1, SCHEMA.getFieldNumber(AccessExpressionMarkings.COLUMN_VISIBILITY_KEY)); + } + + @Test + public void testSchemaGetFieldNumberUnknown() { + assertEquals(0, SCHEMA.getFieldNumber("nonExistentField")); + } + + @Test + public void testSchemaIsInitialized() { + assertTrue(SCHEMA.isInitialized(AccessExpressionMarkings.create("A"))); + assertTrue(SCHEMA.isInitialized(AccessExpressionMarkings.builder().build())); + } + + @Test + public void testSchemaNewMessage() { + AccessExpressionMarkings msg = SCHEMA.newMessage(); + assertNotNull(msg); + assertTrue(msg.isEmpty()); + } + + @Test + public void testSchemaMessageName() { + assertEquals("AccessExpressionMarkings", SCHEMA.messageName()); + } + + @Test + public void testSchemaMessageFullName() { + assertEquals("datawave.marking.AccessExpressionMarkings", SCHEMA.messageFullName()); + } + + @Test + public void testSchemaTypeClass() { + assertEquals(AccessExpressionMarkings.class, SCHEMA.typeClass()); + } + + // -- Protostuff Schema round-trip serialization -- + + @Test + public void testProtostuffRoundTrip() { + AccessExpressionMarkings original = AccessExpressionMarkings.create("(A&B)|C"); + AccessExpressionMarkings deserialized = makeProtostuffRoundTrip(original); + assertEquals(original.getColumnVisibilityString(), deserialized.getColumnVisibilityString()); + } + + @Test + public void testProtostuffRoundTripEmpty() { + AccessExpressionMarkings original = AccessExpressionMarkings.builder().build(); + AccessExpressionMarkings deserialized = makeProtostuffRoundTrip(original); + assertTrue(deserialized.isEmpty()); + } + + @Test + public void testProtostuffRoundTripComplexExpression() { + AccessExpressionMarkings original = AccessExpressionMarkings.create("(A&B)|(C&D&E)"); + AccessExpressionMarkings deserialized = makeProtostuffRoundTrip(original); + assertEquals("(A&B)|(C&D&E)", deserialized.getColumnVisibilityString()); + assertFalse(deserialized.isEmpty()); + } + + private static String makeJaxbMarshall(AccessExpressionMarkings original) throws JAXBException { + JAXBContext ctx = JAXBContext.newInstance(AccessExpressionMarkings.class); + ByteArrayOutputStream out = marshal(original, ctx); + return out.toString(); + } + + private static AccessExpressionMarkings makeJaxbUnmarshall(String expression) throws JAXBException, IOException { + JAXBContext ctx = JAXBContext.newInstance(AccessExpressionMarkings.class); + + String xml = String.format( + "" + + "%s", + expression); + + JAXBElement element; + try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { + out.write(xml.getBytes()); + element = unmarshal(out, ctx); + } + return element.getValue(); + } + + private static AccessExpressionMarkings makeJaxbRoundTrip(AccessExpressionMarkings original) throws JAXBException { + JAXBContext ctx = JAXBContext.newInstance(AccessExpressionMarkings.class); + ByteArrayOutputStream out = marshal(original, ctx); + JAXBElement result = unmarshal(out, ctx); + return result.getValue(); + } + + private static ByteArrayOutputStream marshal(AccessExpressionMarkings original, JAXBContext ctx) throws JAXBException { + Marshaller marshaller = ctx.createMarshaller(); + JAXBElement element = new JAXBElement<>(new QName("http://webservice.datawave.nsa/v1", "markings"), + AccessExpressionMarkings.class, original); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + marshaller.marshal(element, out); + return out; + } + + private static JAXBElement unmarshal(ByteArrayOutputStream out, JAXBContext ctx) throws JAXBException { + Unmarshaller unmarshaller = ctx.createUnmarshaller(); + return unmarshaller.unmarshal(new StreamSource(new ByteArrayInputStream(out.toByteArray())), AccessExpressionMarkings.class); + } + + private static AccessExpressionMarkings makeProtostuffRoundTrip(AccessExpressionMarkings original) { + LinkedBuffer buffer = LinkedBuffer.allocate(); + byte[] bytes = ProtostuffIOUtil.toByteArray(original, SCHEMA, buffer); + + AccessExpressionMarkings deserialized = SCHEMA.newMessage(); + ProtostuffIOUtil.mergeFrom(bytes, deserialized, SCHEMA); + return deserialized; + } + +} diff --git a/core/utils/accumulo-utils/src/test/java/datawave/marking/AccessExpressionUtilTest.java b/core/utils/accumulo-utils/src/test/java/datawave/marking/AccessExpressionUtilTest.java new file mode 100644 index 00000000000..fe598ca3d12 --- /dev/null +++ b/core/utils/accumulo-utils/src/test/java/datawave/marking/AccessExpressionUtilTest.java @@ -0,0 +1,215 @@ +package datawave.marking; + +import static datawave.marking.AccessExpressionMarkings.ACCESS; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.apache.accumulo.access.AccessExpression; +import org.apache.accumulo.access.ParsedAccessExpression; +import org.apache.accumulo.core.security.ColumnVisibility; +import org.junit.jupiter.api.Test; + +public class AccessExpressionUtilTest { + + // -- toColumnVisibility -- + + @Test + public void testToColumnVisibilityNull() { + assertNull(AccessExpressionUtil.toColumnVisibility(null)); + } + + @Test + public void testToColumnVisibilityEmpty() { + AccessExpression ae = ACCESS.newExpression(""); + ColumnVisibility cv = AccessExpressionUtil.toColumnVisibility(ae); + assertEquals(0, cv.getExpression().length); + } + + @Test + public void testToColumnVisibilitySimple() { + AccessExpression ae = ACCESS.newExpression("A&B"); + ColumnVisibility cv = AccessExpressionUtil.toColumnVisibility(ae); + assertEquals("A&B", new String(cv.getExpression())); + } + + // -- toAccessExpression(ColumnVisibility) -- + + @Test + public void testToAccessExpressionFromColumnVisibilityNull() { + assertNull(AccessExpressionUtil.toAccessExpression((ColumnVisibility) null)); + } + + @Test + public void testToAccessExpressionFromColumnVisibilityEmpty() { + ColumnVisibility cv = new ColumnVisibility(); + AccessExpression ae = AccessExpressionUtil.toAccessExpression(cv); + assertEquals("", ae.getExpression()); + } + + @Test + public void testToAccessExpressionFromColumnVisibility() { + ColumnVisibility cv = new ColumnVisibility("A|B"); + AccessExpression ae = AccessExpressionUtil.toAccessExpression(cv); + assertEquals("A|B", ae.getExpression()); + } + + // -- toAccessExpression(String) -- + + @Test + public void testToAccessExpressionFromStringNull() { + AccessExpression ae = AccessExpressionUtil.toAccessExpression((String) null); + assertEquals("", ae.getExpression()); + } + + @Test + public void testToAccessExpressionFromStringEmpty() { + AccessExpression ae = AccessExpressionUtil.toAccessExpression(""); + assertEquals("", ae.getExpression()); + } + + @Test + public void testToAccessExpressionFromString() { + AccessExpression ae = AccessExpressionUtil.toAccessExpression("X&Y"); + assertEquals("X&Y", ae.getExpression()); + } + + // -- toAccessExpression(byte[]) -- + + @Test + public void testToAccessExpressionFromBytesNull() { + AccessExpression ae = AccessExpressionUtil.toAccessExpression((byte[]) null); + assertEquals("", ae.getExpression()); + } + + @Test + public void testToAccessExpressionFromBytesEmpty() { + AccessExpression ae = AccessExpressionUtil.toAccessExpression(new byte[0]); + assertEquals("", ae.getExpression()); + } + + @Test + public void testToAccessExpressionFromBytes() { + AccessExpression ae = AccessExpressionUtil.toAccessExpression("C&D".getBytes()); + assertEquals("C&D", ae.getExpression()); + } + + // -- normalize -- + + @Test + public void testNormalizeEmpty() { + AccessExpression ae = ACCESS.newExpression(""); + AccessExpression normalized = AccessExpressionUtil.normalize(ae); + assertEquals("", normalized.getExpression()); + } + + @Test + public void testNormalizeSortsTerms() { + AccessExpression ae = ACCESS.newExpression("Z&Y&X"); + AccessExpression normalized = AccessExpressionUtil.normalize(ae); + assertEquals("X&Y&Z", normalized.getExpression()); + } + + @Test + public void testNormalizeDeduplicates() { + AccessExpression ae = ACCESS.newExpression("A&B&A"); + AccessExpression normalized = AccessExpressionUtil.normalize(ae); + assertEquals("A&B", normalized.getExpression()); + } + + @Test + public void testNormalizeFlattensNestedAnds() { + AccessExpression ae = ACCESS.newExpression("A&(B&C)"); + AccessExpression normalized = AccessExpressionUtil.normalize(ae); + assertEquals("A&B&C", normalized.getExpression()); + } + + @Test + public void testNormalizeSortsOrSubExpressions() { + AccessExpression ae = ACCESS.newExpression("(Z&Y)|(C&B)"); + AccessExpression normalized = AccessExpressionUtil.normalize(ae); + assertEquals("(B&C)|(Y&Z)", normalized.getExpression()); + } + + @Test + public void testNormalizeRemovesUnnecessaryQuotes() { + AccessExpression ae = ACCESS.newExpression("\"ABC\"&\"XYZ\""); + AccessExpression normalized = AccessExpressionUtil.normalize(ae); + assertEquals("ABC&XYZ", normalized.getExpression()); + } + + @Test + public void testNormalizeRemovesUnnecessaryParentheses() { + AccessExpression ae = ACCESS.newExpression("(((ABC)|(XYZ)))"); + AccessExpression normalized = AccessExpressionUtil.normalize(ae); + assertEquals("ABC|XYZ", normalized.getExpression()); + } + + @Test + public void testNormalizeSingleAuth() { + AccessExpression ae = ACCESS.newExpression("ALPHA"); + AccessExpression normalized = AccessExpressionUtil.normalize(ae); + assertEquals("ALPHA", normalized.getExpression()); + } + + @Test + public void testNormalizeComplex1() { + AccessExpression ae = ACCESS.newExpression("(A&A)|(B&B)|(A&A&A)"); + AccessExpression normalized = AccessExpressionUtil.normalize(ae); + assertEquals("A|B", normalized.getExpression()); + } + + @Test + public void testNormalizeComplex2() { + AccessExpression ae = ACCESS.newExpression("(Y|B|Y)&(Z|A|Z)"); + AccessExpression normalized = AccessExpressionUtil.normalize(ae); + assertEquals("(A|Z)&(B|Y)", normalized.getExpression()); + } + + @Test + public void testNormalizeAlreadyOptimal() { + AccessExpression ae = ACCESS.newExpression("(A|B|C)&(B|C|D)"); + AccessExpression normalized = AccessExpressionUtil.normalize(ae); + assertEquals("(A|B|C)&(B|C|D)", normalized.getExpression()); + } + + // -- emptyExpression / getAccess -- + + @Test + public void testEmptyExpression() { + assertEquals("", AccessExpressionUtil.emptyExpression().getExpression()); + } + + @Test + public void testGetAccess() { + assertEquals(ACCESS, AccessExpressionUtil.getAccess()); + } + + // -- round-trip -- + + @Test + public void testRoundTripColumnVisibilityToAccessExpression() { + ColumnVisibility original = new ColumnVisibility("A&B&C"); + AccessExpression ae = AccessExpressionUtil.toAccessExpression(original); + ColumnVisibility roundTripped = AccessExpressionUtil.toColumnVisibility(ae); + assertEquals(new String(original.getExpression()), new String(roundTripped.getExpression())); + } + + // -- NormalizedExpression -- + + @Test + public void testNormalizedExpressionCompareTo() { + var a = new AccessExpressionUtil.NormalizedExpression("A", ParsedAccessExpression.ExpressionType.AUTHORIZATION); + var b = new AccessExpressionUtil.NormalizedExpression("B", ParsedAccessExpression.ExpressionType.AUTHORIZATION); + assertTrue(a.compareTo(b) < 0); + assertTrue(b.compareTo(a) > 0); + assertEquals(0, a.compareTo(a)); + } + + @Test + public void testNormalizedExpressionEquals() { + var a = new AccessExpressionUtil.NormalizedExpression("A", ParsedAccessExpression.ExpressionType.AUTHORIZATION); + var aDuplicate = new AccessExpressionUtil.NormalizedExpression("A", ParsedAccessExpression.ExpressionType.AUTHORIZATION); + assertEquals(a, aDuplicate); + } +} diff --git a/core/utils/accumulo-utils/src/test/java/datawave/marking/MarkingFunctionsDefaultCombineTest.java b/core/utils/accumulo-utils/src/test/java/datawave/marking/MarkingFunctionsDefaultCombineTest.java index 7b23d454635..eaeb4dfd987 100644 --- a/core/utils/accumulo-utils/src/test/java/datawave/marking/MarkingFunctionsDefaultCombineTest.java +++ b/core/utils/accumulo-utils/src/test/java/datawave/marking/MarkingFunctionsDefaultCombineTest.java @@ -1,61 +1,99 @@ package datawave.marking; -import static datawave.marking.MarkingFunctions.Default.COLUMN_VISIBILITY; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; -import java.util.Map; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; import org.apache.accumulo.core.security.ColumnVisibility; import org.junit.jupiter.api.Test; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Sets; - public class MarkingFunctionsDefaultCombineTest { @Test public void testCombineAnds() throws MarkingFunctions.Exception { - MarkingFunctions markingFunctions = new MarkingFunctions.Default(); + MarkingFunctions markingFunctions = new MarkingFunctions.Default(); - ColumnVisibility oneAnna = new ColumnVisibility("A&B"); - ColumnVisibility twoAnna = new ColumnVisibility("A&C"); + AccessExpressionMarkings oneAnna = AccessExpressionMarkings.builder().columnVisibility("A&B").build(); + AccessExpressionMarkings twoAnna = AccessExpressionMarkings.builder().columnVisibility("A&C").build(); - ColumnVisibility combined = markingFunctions.combine(Sets.newHashSet(oneAnna, twoAnna)); - assertEquals(new ColumnVisibility("A&B&C"), combined); + AccessExpressionMarkings combined = markingFunctions.combine(List.of(oneAnna, twoAnna)); + assertEquals(AccessExpressionMarkings.builder().columnVisibility("A&B&C").build(), combined); } @Test public void testCombineOrs() throws MarkingFunctions.Exception { - MarkingFunctions markingFunctions = new MarkingFunctions.Default(); + MarkingFunctions markingFunctions = new MarkingFunctions.Default(); - ColumnVisibility oneOr = new ColumnVisibility("A|B"); - ColumnVisibility twoOr = new ColumnVisibility("A|C"); + AccessExpressionMarkings oneOr = AccessExpressionMarkings.builder().columnVisibility("A|B").build(); + AccessExpressionMarkings twoOr = AccessExpressionMarkings.builder().columnVisibility("A|C").build(); - ColumnVisibility combined = markingFunctions.combine(Sets.newHashSet(oneOr, twoOr)); - assertEquals(new ColumnVisibility("(A|B)&(A|C)"), combined); + AccessExpressionMarkings combined = markingFunctions.combine(List.of(oneOr, twoOr)); + assertEquals(AccessExpressionMarkings.builder().columnVisibility("(A|B)&(A|C)").build(), combined); } @Test public void testCombineMapsOfAnds() throws MarkingFunctions.Exception { - MarkingFunctions markingFunctions = new MarkingFunctions.Default(); - - Map mapOne = ImmutableMap.of(COLUMN_VISIBILITY, "A&B"); - Map mapTwo = ImmutableMap.of(COLUMN_VISIBILITY, "A&C"); + MarkingFunctions markingFunctions = new MarkingFunctions.Default(); - Map expected = ImmutableMap.of(COLUMN_VISIBILITY, "A&B&C"); + AccessExpressionMarkings cv1 = AccessExpressionMarkings.builder().columnVisibility("A&B").build(); + AccessExpressionMarkings cv2 = AccessExpressionMarkings.builder().columnVisibility("A&C").build(); + AccessExpressionMarkings expected = AccessExpressionMarkings.builder().columnVisibility("A&B&C").build(); - assertEquals(expected, markingFunctions.combine(mapOne, mapTwo)); + assertEquals(expected, markingFunctions.combine(List.of(cv1, cv2))); } @Test public void testCombineMapsOfOrs() throws MarkingFunctions.Exception { - MarkingFunctions markingFunctions = new MarkingFunctions.Default(); + MarkingFunctions markingFunctions = new MarkingFunctions.Default(); - Map mapOne = ImmutableMap.of(COLUMN_VISIBILITY, "A|B"); - Map mapTwo = ImmutableMap.of(COLUMN_VISIBILITY, "A|C"); + AccessExpressionMarkings cv1 = AccessExpressionMarkings.builder().columnVisibility("A|B").build(); + AccessExpressionMarkings cv2 = AccessExpressionMarkings.builder().columnVisibility("A|C").build(); + AccessExpressionMarkings expected = AccessExpressionMarkings.builder().columnVisibility("(A|B)&(A|C)").build(); - Map expected = ImmutableMap.of(COLUMN_VISIBILITY, "(A|B)&(A|C)"); + assertEquals(expected, markingFunctions.combine(List.of(cv1, cv2))); + } + + @Test + public void testCombineSkipsNullMarkings() throws MarkingFunctions.Exception { + MarkingFunctions markingFunctions = new MarkingFunctions.Default(); + + AccessExpressionMarkings aem1 = AccessExpressionMarkings.builder().columnVisibility("A&B").build(); + AccessExpressionMarkings aem2 = AccessExpressionMarkings.builder().columnVisibility("A&C").build(); + AccessExpressionMarkings expected = AccessExpressionMarkings.builder().columnVisibility("A&B&C").build(); + + Collection> markings = new ArrayList<>(Arrays.asList(aem1, null, aem2, null)); + assertEquals(expected, markingFunctions.combine(markings)); + } + + @Test + public void testCombineAllNullMarkingsReturnsNull() throws MarkingFunctions.Exception { + MarkingFunctions markingFunctions = new MarkingFunctions.Default(); + + Collection> markings = new ArrayList<>(Arrays.asList(null, null)); + assertNull(markingFunctions.combine(markings)); + } + + @Test + public void testCombineVisibilitiesSkipsNullElements() throws MarkingFunctions.Exception { + MarkingFunctions markingFunctions = new MarkingFunctions.Default(); + + ColumnVisibility vis1 = new ColumnVisibility("A&B"); + ColumnVisibility vis2 = new ColumnVisibility("A&C"); + + Collection visibilities = new ArrayList<>(Arrays.asList(vis1, null, vis2, null)); + ColumnVisibility combined = markingFunctions.combineVisibilities(visibilities); + assertEquals(new ColumnVisibility("A&B&C"), combined); + } + + @Test + public void testCombineVisibilitiesAllNullsReturnsEmpty() throws MarkingFunctions.Exception { + MarkingFunctions markingFunctions = new MarkingFunctions.Default(); - assertEquals(expected, markingFunctions.combine(mapOne, mapTwo)); + Collection visibilities = new ArrayList<>(Arrays.asList(null, null)); + assertEquals(new ColumnVisibility(), markingFunctions.combineVisibilities(visibilities)); } } diff --git a/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java b/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java index b5dfaca8076..d927be96c7f 100644 --- a/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java +++ b/core/utils/metadata-utils/src/main/java/datawave/iterators/FrequencyMetadataAggregator.java @@ -54,7 +54,7 @@ public class FrequencyMetadataAggregator extends WrappingIterator implements Opt private static final Logger log = Logger.getLogger(FrequencyMetadataAggregator.class); private static final String NULL_BYTE = "\0"; - private static final MarkingFunctions markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); + private final MarkingFunctions markingFunctions; private boolean combineVisibilities; private String columnsOption; @@ -82,6 +82,7 @@ public FrequencyMetadataAggregator() { cache = new TreeMap<>(); visibilityToDateFrequencies = new HashMap<>(); visibilityToMaxTimestamp = new HashMap<>(); + markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); } @Override @@ -584,9 +585,8 @@ private Map buildCacheEntries() { private ColumnVisibility combineAllVisibilities() { Set visibilities = visibilityToDateFrequencies.keySet(); try { - return markingFunctions.combine(visibilities); + return markingFunctions.combineVisibilities(visibilities); } catch (MarkingFunctions.Exception e) { - log.error("Failed to combine visibilities " + visibilities); throw new IllegalArgumentException("Failed to combine visibilities " + visibilities, e); } } diff --git a/core/utils/metadata-utils/src/main/java/datawave/query/model/ModelKeyParser.java b/core/utils/metadata-utils/src/main/java/datawave/query/model/ModelKeyParser.java index 1a4f9a01a9b..f2521e31175 100644 --- a/core/utils/metadata-utils/src/main/java/datawave/query/model/ModelKeyParser.java +++ b/core/utils/metadata-utils/src/main/java/datawave/query/model/ModelKeyParser.java @@ -127,6 +127,9 @@ public static FieldMapping parseKey(Key key, Value value) { } public static Key createKey(FieldMapping mapping, String modelName) { + if (null == mapping.getColumnVisibility()) { + throw new RuntimeException("ColumnVisibility was null, unable to create Accumulo mutation"); + } ColumnVisibility cv = new ColumnVisibility(mapping.getColumnVisibility()); String dataType = StringUtils.isEmpty(mapping.getDatatype()) ? "" : NULL_BYTE + mapping.getDatatype().trim(); @@ -185,6 +188,9 @@ private static Set getAttrValues(Set attributes) { } public static Mutation createMutation(FieldMapping mapping, String modelName) { + if (null == mapping.getColumnVisibility()) { + throw new RuntimeException("ColumnVisibility was null, unable to create Accumulo mutation"); + } ColumnVisibility cv = new ColumnVisibility(mapping.getColumnVisibility()); Mutation m; String dataType = StringUtils.isEmpty(mapping.getDatatype()) ? "" : NULL_BYTE + mapping.getDatatype().trim(); @@ -208,6 +214,9 @@ public static Mutation createMutation(FieldMapping mapping, String modelName) { } public static Mutation createDeleteMutation(FieldMapping mapping, String modelName) { + if (null == mapping.getColumnVisibility()) { + throw new RuntimeException("ColumnVisibility was null, unable to create Accumulo mutation"); + } ColumnVisibility cv = new ColumnVisibility(mapping.getColumnVisibility()); Mutation m; String dataType = StringUtils.isEmpty(mapping.getDatatype()) ? "" : NULL_BYTE + mapping.getDatatype().trim(); diff --git a/import-control-accumulo.xml b/import-control-accumulo.xml index dd6e8eb2ca9..d697b47a953 100644 --- a/import-control-accumulo.xml +++ b/import-control-accumulo.xml @@ -10,6 +10,7 @@ |datawave\.test.*| |gov.nsa.datawave.*)" regex="true" strategyOnMismatch="allowed"> + diff --git a/microservices/services/accumulo/api/pom.xml b/microservices/services/accumulo/api/pom.xml index aa42f6e2c52..677716a48a8 100644 --- a/microservices/services/accumulo/api/pom.xml +++ b/microservices/services/accumulo/api/pom.xml @@ -19,7 +19,7 @@ http://webservice.datawave.nsa/v1 1.17.1 - 7.33.1 + 7.40.0-SNAPSHOT 2.14.3 2.3.3 @@ -51,12 +51,6 @@ gov.nsa.datawave.core accumulo-utils ${version.datawave} - - - com.google.guava - guava - - gov.nsa.datawave.core diff --git a/microservices/services/accumulo/api/src/main/java/datawave/webservice/response/objects/DefaultKey.java b/microservices/services/accumulo/api/src/main/java/datawave/webservice/response/objects/DefaultKey.java index 5fd21cff9c6..54220618249 100644 --- a/microservices/services/accumulo/api/src/main/java/datawave/webservice/response/objects/DefaultKey.java +++ b/microservices/services/accumulo/api/src/main/java/datawave/webservice/response/objects/DefaultKey.java @@ -2,16 +2,15 @@ import static java.nio.charset.StandardCharsets.UTF_8; -import java.util.HashMap; -import java.util.Map; - import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlElement; +import org.apache.accumulo.access.AccessExpression; import org.apache.commons.codec.binary.Base64; -import datawave.marking.MarkingFunctions; +import datawave.marking.AccessExpressionMarkings; +import datawave.marking.Markings; import datawave.webservice.query.util.TypedValue; @XmlAccessorType(XmlAccessType.NONE) @@ -87,18 +86,14 @@ public long getTimestamp() { } @Override - public void setMarkings(Map markings) { - HashMap markingMap = new HashMap<>(); - if (markings != null) { - markingMap.putAll(markings); - } - this.markings = markingMap; - - setColumnVisibility(markingMap.getOrDefault(MarkingFunctions.Default.COLUMN_VISIBILITY, "")); + public void setMarkings(Markings markings) { + this.markings = markings; + AccessExpression ae = markings.toAccessExpression(); + this.columnVisibility = new TypedValue(ae != null ? ae.getExpression() : ""); } @Override - public Map getMarkings() { + public Markings getMarkings() { return this.markings; } @@ -116,11 +111,8 @@ public void setColQual(String colQual) { public void setColumnVisibility(String colVis) { this.columnVisibility = (colVis == null) ? new TypedValue("") : new TypedValue(colVis); - if (markings == null) { - markings = new HashMap<>(); - } - markings.put(MarkingFunctions.Default.COLUMN_VISIBILITY, this.columnVisibility.getValue().toString()); - + String expr = this.columnVisibility.getValue().toString(); + this.markings = AccessExpressionMarkings.builder().columnVisibility(expr).build(); } public void setTimestamp(long timestamp) { diff --git a/microservices/services/accumulo/api/src/main/java/datawave/webservice/response/objects/KeyBase.java b/microservices/services/accumulo/api/src/main/java/datawave/webservice/response/objects/KeyBase.java index 7b414c410d2..1d4b04aa977 100644 --- a/microservices/services/accumulo/api/src/main/java/datawave/webservice/response/objects/KeyBase.java +++ b/microservices/services/accumulo/api/src/main/java/datawave/webservice/response/objects/KeyBase.java @@ -1,18 +1,17 @@ package datawave.webservice.response.objects; -import java.util.Map; - import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlSeeAlso; +import datawave.marking.Markings; import datawave.webservice.query.result.event.HasMarkings; @XmlAccessorType(XmlAccessType.NONE) @XmlSeeAlso(DefaultKey.class) public abstract class KeyBase implements HasMarkings { - protected Map markings; + protected Markings markings; public abstract String getRow(); diff --git a/microservices/services/accumulo/api/src/main/java/datawave/webservice/response/objects/Visibility.java b/microservices/services/accumulo/api/src/main/java/datawave/webservice/response/objects/Visibility.java index 4fd67012dac..d93748785eb 100644 --- a/microservices/services/accumulo/api/src/main/java/datawave/webservice/response/objects/Visibility.java +++ b/microservices/services/accumulo/api/src/main/java/datawave/webservice/response/objects/Visibility.java @@ -1,17 +1,13 @@ package datawave.webservice.response.objects; -import java.util.HashMap; -import java.util.Map; - import javax.xml.bind.annotation.XmlAccessOrder; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorOrder; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlAttribute; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; +import javax.xml.bind.annotation.XmlTransient; -import datawave.webservice.xml.util.StringMapAdapter; +import datawave.marking.Markings; @XmlAccessorType(XmlAccessType.FIELD) @XmlAccessorOrder(XmlAccessOrder.ALPHABETICAL) @@ -20,9 +16,8 @@ public class Visibility { @XmlAttribute(name = "valid") private Boolean valid; - @XmlElement(name = "markings") - @XmlJavaTypeAdapter(StringMapAdapter.class) - private HashMap markings; + @XmlTransient + private Markings markings; @XmlAttribute(name = "visibility") private String visibility; @@ -43,16 +38,12 @@ public void setVisibility(String visibility) { this.visibility = visibility; } - public Map getMarkings() { + public Markings getMarkings() { return markings; } - public void setMarkings(Map markings) { - if (this.markings == null) { - this.markings = new HashMap(); - } - this.markings.clear(); - this.markings.putAll(markings); + public void setMarkings(Markings markings) { + this.markings = markings; } } diff --git a/microservices/services/accumulo/api/src/test/java/datawave/webservice/response/LookupResponseTest.java b/microservices/services/accumulo/api/src/test/java/datawave/webservice/response/LookupResponseTest.java index 6b5f6bfc993..e6f43a1f261 100644 --- a/microservices/services/accumulo/api/src/test/java/datawave/webservice/response/LookupResponseTest.java +++ b/microservices/services/accumulo/api/src/test/java/datawave/webservice/response/LookupResponseTest.java @@ -6,9 +6,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -19,6 +17,7 @@ import com.fasterxml.jackson.databind.type.TypeFactory; import com.fasterxml.jackson.module.jaxb.JaxbAnnotationIntrospector; +import datawave.marking.AccessExpressionMarkings; import datawave.webservice.query.exception.QueryExceptionType; import datawave.webservice.response.objects.DefaultKey; import datawave.webservice.response.objects.Entry; @@ -39,8 +38,7 @@ private void responseSetup() { lookupResponse = new LookupResponse(); List entryList = new ArrayList<>(); for (int i = 0; i < 3; i++) { - Map markings = new HashMap<>(); - markings.put("columnVisibility", "A&B&C" + i); + AccessExpressionMarkings markings = AccessExpressionMarkings.builder().columnVisibility("A&B&C" + i).build(); KeyBase responseKey = new DefaultKey(); responseKey.setRow("row" + i); responseKey.setColFam("cf" + i); diff --git a/microservices/services/accumulo/service/pom.xml b/microservices/services/accumulo/service/pom.xml index fca3f5ee1f8..33f6ee21135 100644 --- a/microservices/services/accumulo/service/pom.xml +++ b/microservices/services/accumulo/service/pom.xml @@ -24,12 +24,12 @@ 3.20.0 5.2.0 5.2.0 - 7.33.1 - 4.0.1 + 7.40.0-SNAPSHOT + 4.0.2-SNAPSHOT 4.0.2 4.0.2 - 4.0.7 - 4.0.5 + 4.0.8-SNAPSHOT + 4.0.6-SNAPSHOT 31.1-jre 3.3.4 4.0.4 @@ -69,6 +69,11 @@ base-rest-responses ${version.datawave} + + gov.nsa.datawave.core + datawave-core-common-util + ${version.datawave} + gov.nsa.datawave.core type-utils @@ -229,6 +234,10 @@ gov.nsa.datawave.core base-rest-responses + + gov.nsa.datawave.core + datawave-core-common-util + gov.nsa.datawave.core type-utils diff --git a/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/admin/AdminService.java b/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/admin/AdminService.java index 748d809785b..f33bc1707a7 100644 --- a/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/admin/AdminService.java +++ b/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/admin/AdminService.java @@ -42,6 +42,7 @@ import org.springframework.stereotype.Service; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.webservice.query.util.OptionallyEncodedString; import datawave.webservice.request.UpdateRequest; import datawave.webservice.request.objects.MutationEntry; @@ -68,11 +69,11 @@ public class AdminService { private final Logger log = LoggerFactory.getLogger(this.getClass()); - private final MarkingFunctions markingFunctions; + private final MarkingFunctions markingFunctions; private final AccumuloClient warehouseAccumuloClient; @Autowired - public AdminService(@Qualifier("warehouse") AccumuloClient warehouseAccumuloClient, MarkingFunctions markingFunctions) { + public AdminService(@Qualifier("warehouse") AccumuloClient warehouseAccumuloClient, MarkingFunctions markingFunctions) { this.warehouseAccumuloClient = warehouseAccumuloClient; this.markingFunctions = markingFunctions; } @@ -569,7 +570,7 @@ public ValidateVisibilityResponse validateVisibilities(String[] visibilityArray) vis.setValid(false); visibilityList.add(vis); try { - Map markings = markingFunctions.translateFromColumnVisibilityForAuths(new ColumnVisibility(v), authorizations); + Markings markings = markingFunctions.translateFromColumnVisibilityForAuths(new ColumnVisibility(v), authorizations); vis.setVisibility(v); vis.setValid(true); vis.setMarkings(markings); diff --git a/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/config/AccumuloConfiguration.java b/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/config/AccumuloConfiguration.java index 7a4ac5cf8a4..48a0f79dcb0 100644 --- a/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/config/AccumuloConfiguration.java +++ b/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/config/AccumuloConfiguration.java @@ -37,7 +37,7 @@ public AccumuloProperties metricsAccumuloProperies(MetricsClusterProperties metr @Bean @ConditionalOnMissingBean - public MarkingFunctions markingFunctions() { + public MarkingFunctions markingFunctions() { return new MarkingFunctions.Default(); } diff --git a/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/lookup/LookupService.java b/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/lookup/LookupService.java index 26b89ed9ef0..1677af0f93a 100644 --- a/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/lookup/LookupService.java +++ b/microservices/services/accumulo/service/src/main/java/datawave/microservice/accumulo/lookup/LookupService.java @@ -36,6 +36,7 @@ import datawave.accumulo.util.security.UserAuthFunctions; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.marking.SecurityMarking; import datawave.microservice.accumulo.lookup.config.LookupAuditProperties; import datawave.microservice.accumulo.lookup.config.LookupProperties; @@ -82,7 +83,7 @@ public interface Parameter { } private final SecurityMarking auditSecurityMarking; - private final MarkingFunctions markingFunctions; + private final MarkingFunctions markingFunctions; private final AccumuloClient connection; private final LookupAuditProperties lookupAuditProperties; private final LookupProperties lookupProperties; @@ -97,7 +98,7 @@ public interface Parameter { public LookupService( @Qualifier("auditLookupSecurityMarking") SecurityMarking auditSecurityMarking, - MarkingFunctions markingFunctions, + MarkingFunctions markingFunctions, @Qualifier("warehouse") AccumuloClient connection, LookupAuditProperties lookupAuditProperties, @@ -218,12 +219,7 @@ public LookupResponse lookup(LookupRequest request, DatawaveUserDetails currentU continue; } - //@formatter:off - final Map markings = markingFunctions.translateFromColumnVisibilityForAuths( - new ColumnVisibility(k.getColumnVisibility()), - mergedAuths - ); - //@formatter:on + final Markings markings = markingFunctions.translateFromColumnVisibilityForAuths(new ColumnVisibility(k.getColumnVisibility()), mergedAuths); final KeyBase responseKey = responseObjectFactory.createKey(); responseKey.setRow(currRow); diff --git a/microservices/services/dictionary/api/pom.xml b/microservices/services/dictionary/api/pom.xml index 5fa0f3e29a2..27c0cdf018e 100644 --- a/microservices/services/dictionary/api/pom.xml +++ b/microservices/services/dictionary/api/pom.xml @@ -19,7 +19,7 @@ http://webservice.datawave.nsa/v1 3.20.0 - 7.33.1 + 7.40.0-SNAPSHOT 31.1-jre 2.3.3 1.6.2 @@ -31,6 +31,11 @@ guava ${version.guava} + + gov.nsa.datawave.core + accumulo-utils + ${version.datawave} + gov.nsa.datawave.core base-rest-responses @@ -69,6 +74,10 @@ com.google.guava guava + + gov.nsa.datawave.core + accumulo-utils + gov.nsa.datawave.core base-rest-responses diff --git a/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java b/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java index ae9daab5890..3b4f1f8bb47 100644 --- a/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java +++ b/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java @@ -2,17 +2,17 @@ import java.io.IOException; import java.io.Serializable; -import java.util.HashMap; -import java.util.Map; import java.util.Objects; import javax.xml.bind.annotation.XmlAccessOrder; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorOrder; import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlTransient; -import datawave.webservice.query.result.event.MapSchema; +import datawave.marking.AccessExpressionMarkings; +import datawave.marking.Markings; import io.protostuff.Input; import io.protostuff.Message; import io.protostuff.Output; @@ -29,11 +29,12 @@ public DefaultDescription(String description) { this.description = description; } - public DefaultDescription(String description, Map markings) { + public DefaultDescription(String description, Markings markings) { this.description = description; this.markings = markings; } + @XmlElement(name = "description") public String getDescription() { return description; } @@ -43,12 +44,14 @@ public void setDescription(String description) { } @Override - public void setMarkings(Map markings) { - this.markings = markings; + @XmlElement(name = "markings", type = AccessExpressionMarkings.class) + public Markings getMarkings() { + return this.markings; } - public Map getMarkings() { - return this.markings; + @Override + public void setMarkings(Markings markings) { + this.markings = markings; } @Override @@ -86,7 +89,7 @@ public Schema cachedSchema() { } @XmlTransient - private static final Schema SCHEMA = new Schema() { + private static final Schema SCHEMA = new Schema<>() { public DefaultDescription newMessage() { return new DefaultDescription(); } @@ -112,7 +115,7 @@ public void writeTo(Output output, DefaultDescription message) throws IOExceptio output.writeString(1, message.description, false); } if (message.markings != null) - output.writeObject(1, message.markings, MapSchema.SCHEMA, false); + output.writeObject(2, (AccessExpressionMarkings) message.markings, AccessExpressionMarkings.SCHEMA, false); } public void mergeFrom(Input input, DefaultDescription message) throws IOException { @@ -123,8 +126,8 @@ public void mergeFrom(Input input, DefaultDescription message) throws IOExceptio message.description = input.readString(); break; case 2: - message.markings = new HashMap(); - input.mergeObject(message.markings, MapSchema.SCHEMA); + message.markings = AccessExpressionMarkings.builder().build(); + input.mergeObject((AccessExpressionMarkings) message.markings, AccessExpressionMarkings.SCHEMA); break; default: input.handleUnknownField(number, this); diff --git a/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java b/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java index b20e09c82d1..777cfd02c01 100644 --- a/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java +++ b/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java @@ -1,25 +1,18 @@ package datawave.webservice.dictionary.data; -import java.util.Map; - import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; +import datawave.marking.Markings; import datawave.webservice.query.result.event.HasMarkings; -import datawave.webservice.xml.util.StringMapAdapter; import io.protostuff.Message; @XmlAccessorType(XmlAccessType.NONE) public abstract class DescriptionBase implements HasMarkings, Message { - @XmlElement(name = "description") protected String description; - @XmlElement(name = "markings") - @XmlJavaTypeAdapter(StringMapAdapter.class) - protected Map markings; + protected Markings markings; public abstract String getDescription(); diff --git a/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/edge/DefaultEdgeDictionary.java b/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/edge/DefaultEdgeDictionary.java index db34844f810..4a4f91c80dd 100644 --- a/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/edge/DefaultEdgeDictionary.java +++ b/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/edge/DefaultEdgeDictionary.java @@ -24,6 +24,7 @@ import io.protostuff.Message; import io.protostuff.Output; import io.protostuff.Schema; +import lombok.Setter; @XmlRootElement(name = "EdgeDictionary") @XmlAccessorType(XmlAccessType.NONE) @@ -33,6 +34,7 @@ public class DefaultEdgeDictionary extends EdgeDictionaryBase getSchema() { return SCHEMA; } @@ -227,7 +225,7 @@ public String getMainContent() { fieldBuilder.append(field).append(SEP); } - String fieldNames = fieldBuilder.toString().substring(0, fieldBuilder.length() - 2); + String fieldNames = fieldBuilder.substring(0, fieldBuilder.length() - 2); String date = metadata.getStartDate(); builder.append("").append(type).append(""); diff --git a/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/edge/MetadataBase.java b/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/edge/MetadataBase.java index c134d79ab3f..926f8e9c779 100644 --- a/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/edge/MetadataBase.java +++ b/microservices/services/dictionary/api/src/main/java/datawave/webservice/dictionary/edge/MetadataBase.java @@ -1,14 +1,12 @@ package datawave.webservice.dictionary.edge; import java.util.List; -import java.util.Map; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlSeeAlso; -import com.google.common.collect.Maps; - +import datawave.marking.Markings; import datawave.webservice.query.result.event.HasMarkings; import io.protostuff.Message; @@ -16,19 +14,13 @@ @XmlSeeAlso(DefaultMetadata.class) public abstract class MetadataBase implements HasMarkings, Message { - protected transient Map markings; + protected transient Markings markings; - public Map getMarkings() { - assureMarkings(); + public Markings getMarkings() { return markings; } - protected void assureMarkings() { - if (this.markings == null) - this.markings = Maps.newHashMap(); - } - - public void setMarkings(Map markings) { + public void setMarkings(Markings markings) { this.markings = markings; } diff --git a/microservices/services/dictionary/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java b/microservices/services/dictionary/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java index 44cecce2691..0d93ab22039 100644 --- a/microservices/services/dictionary/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java +++ b/microservices/services/dictionary/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java @@ -7,7 +7,6 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; import javax.xml.bind.annotation.XmlAccessType; @@ -21,9 +20,9 @@ import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; +import datawave.marking.AccessExpressionMarkings; import datawave.webservice.dictionary.data.DefaultDescription; import datawave.webservice.dictionary.data.DescriptionBase; -import datawave.webservice.query.result.event.MapSchema; import io.protostuff.Input; import io.protostuff.Message; import io.protostuff.Output; @@ -73,7 +72,7 @@ public class DefaultMetadataField extends MetadataFieldBase cachedSchema() { } @XmlTransient - private static final Schema SCHEMA = new Schema() { + private static final Schema SCHEMA = new Schema<>() { private final HashMap fieldMap = new HashMap<>(); { fieldMap.put("fieldName", 1); @@ -250,8 +249,8 @@ public void mergeFrom(Input input, DefaultMetadataField message) throws IOExcept int size = input.readInt32(); message.descriptions = new HashSet<>(size); for (int i = 0; i < size; i++) { - Map markings = new HashMap<>(); - input.mergeObject(markings, MapSchema.SCHEMA); + AccessExpressionMarkings markings = AccessExpressionMarkings.builder().build(); + input.mergeObject(markings, AccessExpressionMarkings.SCHEMA); message.descriptions.add(new DefaultDescription(input.readString(), markings)); } break; @@ -295,7 +294,7 @@ public void writeTo(Output output, DefaultMetadataField message) throws IOExcept output.writeInt32(9, message.getDescriptions().size(), false); for (DescriptionBase desc : message.getDescriptions()) { output.writeString(9, desc.getDescription(), true); - output.writeObject(9, desc.getMarkings(), MapSchema.SCHEMA, false); + output.writeObject(9, (AccessExpressionMarkings) desc.getMarkings(), AccessExpressionMarkings.SCHEMA, false); } output.writeString(10, message.lastUpdated, false); output.writeBool(11, message.tokenized, false); diff --git a/microservices/services/dictionary/api/src/test/java/datawave/webservice/dictionary/data/DefaultDescriptionTest.java b/microservices/services/dictionary/api/src/test/java/datawave/webservice/dictionary/data/DefaultDescriptionTest.java index a6298a171d9..181028e620e 100644 --- a/microservices/services/dictionary/api/src/test/java/datawave/webservice/dictionary/data/DefaultDescriptionTest.java +++ b/microservices/services/dictionary/api/src/test/java/datawave/webservice/dictionary/data/DefaultDescriptionTest.java @@ -7,10 +7,8 @@ import java.io.ByteArrayOutputStream; import java.lang.annotation.Annotation; import java.util.ArrayList; -import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; import javax.xml.bind.JAXBContext; @@ -21,17 +19,31 @@ import org.junit.jupiter.api.Test; +import datawave.marking.AccessExpressionMarkings; + public class DefaultDescriptionTest { @Test public void testMarshall() throws JAXBException { JAXBContext j = JAXBContext.newInstance(DefaultFields.class); - Map markings = new HashMap<>(); - markings.put("columnVisibility", "PRIVATE"); + DefaultFields dicFieldsList = getDefaultFields(); + + Marshaller m = j.createMarshaller(); + m.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + m.marshal(dicFieldsList, out); + + String result = out.toString(); + + assertTrue(result.contains("PRIVATE")); + assertTrue(result.contains("my description")); + } + private static DefaultFields getDefaultFields() { DefaultDescription desc = new DefaultDescription(); - desc.setMarkings(markings); + desc.setMarkings(AccessExpressionMarkings.create("PRIVATE")); desc.setDescription("my description"); Set descs = new HashSet<>(); @@ -47,18 +59,7 @@ public void testMarshall() throws JAXBException { DefaultFields dicFieldsList = new DefaultFields(); dicFieldsList.setFields(dicFields); - - Marshaller m = j.createMarshaller(); - m.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true); - - ByteArrayOutputStream out = new ByteArrayOutputStream(); - m.marshal(dicFieldsList, out); - - String result = out.toString(); - - assertTrue(result.contains("columnVisibility")); - assertTrue(result.contains("PRIVATE")); - assertTrue(result.contains("my description")); + return dicFieldsList; } @Test @@ -83,10 +84,7 @@ public void testUnmarshall() throws JAXBException { + " \n" + " my description\n" + " \n" - + " \n" - + " columnVisibility\n" - + " PRIVATE\n" - + " \n" + + " PRIVATE\n" + " \n" + " \n" + " myField\n" @@ -107,9 +105,9 @@ public void testUnmarshall() throws JAXBException { assertEquals("myType", f.getDatatype()); for (DefaultDescription d : f.getDescriptions()) { assertEquals("my description", d.getDescription()); - Map m = d.getMarkings(); - assertEquals("PRIVATE", m.get("columnVisibility")); - markingCount = m.size(); + assertTrue(d.getMarkings() instanceof AccessExpressionMarkings); + AccessExpressionMarkings aem = (AccessExpressionMarkings) d.getMarkings(); + assertEquals("PRIVATE", aem.getColumnVisibilityString()); descriptionCount++; } fieldCount++; @@ -117,6 +115,5 @@ public void testUnmarshall() throws JAXBException { assertEquals(1, fieldCount); assertEquals(1, descriptionCount); - assertEquals(1, markingCount); } } diff --git a/microservices/services/dictionary/service/pom.xml b/microservices/services/dictionary/service/pom.xml index 2cb915e104c..4dc778a0b64 100644 --- a/microservices/services/dictionary/service/pom.xml +++ b/microservices/services/dictionary/service/pom.xml @@ -28,10 +28,10 @@ 3.2.2 3.20.0 5.2.0 - 7.33.1 + 7.40.0-SNAPSHOT 4.0.2 - 4.0.6 - 4.0.7 + 4.0.7-SNAPSHOT + 4.0.8-SNAPSHOT 3.0.4 31.1-jre 3.3.4 diff --git a/microservices/services/dictionary/service/src/main/java/datawave/microservice/dictionary/DataDictionaryControllerLogic.java b/microservices/services/dictionary/service/src/main/java/datawave/microservice/dictionary/DataDictionaryControllerLogic.java index d19dc671532..afeb7e6c706 100644 --- a/microservices/services/dictionary/service/src/main/java/datawave/microservice/dictionary/DataDictionaryControllerLogic.java +++ b/microservices/services/dictionary/service/src/main/java/datawave/microservice/dictionary/DataDictionaryControllerLogic.java @@ -14,6 +14,8 @@ import com.google.common.collect.Maps; import com.google.common.collect.Multimap; +import datawave.marking.AccessExpressionMarkings; +import datawave.marking.Markings; import datawave.microservice.AccumuloConnectionService; import datawave.microservice.Connection; import datawave.microservice.authorization.user.DatawaveUserDetails; @@ -167,8 +169,7 @@ public VoidResponse setDescriptionPut(String fieldName, String datatype, String public VoidResponse setDescriptionPost(String fieldName, String datatype, String description, String modelName, String modelTable, String columnVisibility, DatawaveUserDetails currentUser) throws Exception { DESC desc = this.responseObjectFactory.getDescription(); - Map markings = Maps.newHashMap(); - markings.put("columnVisibility", columnVisibility); + Markings markings = AccessExpressionMarkings.builder().columnVisibility(columnVisibility).build(); desc.setMarkings(markings); desc.setDescription(description); @@ -281,8 +282,7 @@ public FIELDS fieldNameDescription(String fieldName, String datatype, String mod */ public VoidResponse deleteDescription(String fieldName, String datatype, String modelName, String modelTable, String columnVisibility, DatawaveUserDetails currentUser) throws Exception { - Map markings = Maps.newHashMap(); - markings.put("columnVisibility", columnVisibility); + Markings markings = AccessExpressionMarkings.builder().columnVisibility(columnVisibility).build(); DESC desc = this.responseObjectFactory.getDescription(); desc.setDescription(""); desc.setMarkings(markings); diff --git a/microservices/services/dictionary/service/src/main/java/datawave/microservice/dictionary/config/DictionaryServiceConfiguration.java b/microservices/services/dictionary/service/src/main/java/datawave/microservice/dictionary/config/DictionaryServiceConfiguration.java index 5cd8815170b..41bc312dc62 100644 --- a/microservices/services/dictionary/service/src/main/java/datawave/microservice/dictionary/config/DictionaryServiceConfiguration.java +++ b/microservices/services/dictionary/service/src/main/java/datawave/microservice/dictionary/config/DictionaryServiceConfiguration.java @@ -46,14 +46,14 @@ public UserAuthFunctions userAuthFunctions() { @Bean @Scope("prototype") @ConditionalOnMissingBean - public MetadataDescriptionsHelper metadataHelperWithDescriptions(MarkingFunctions markingFunctions, + public MetadataDescriptionsHelper metadataHelperWithDescriptions(MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { return new MetadataDescriptionsHelper<>(markingFunctions, responseObjectFactory); } @Bean @ConditionalOnMissingBean - public DataDictionary datawaveDataDictionary(MarkingFunctions markingFunctions, + public DataDictionary datawaveDataDictionary(MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, MetadataHelperFactory metadataHelperFactory, MetadataDescriptionsHelperFactory metadataDescriptionsHelperFactory) { return new DataDictionaryImpl(markingFunctions, responseObjectFactory, metadataHelperFactory, metadataDescriptionsHelperFactory); diff --git a/microservices/services/dictionary/service/src/main/java/datawave/microservice/dictionary/data/DataDictionaryImpl.java b/microservices/services/dictionary/service/src/main/java/datawave/microservice/dictionary/data/DataDictionaryImpl.java index 5a508b37406..d0042199493 100644 --- a/microservices/services/dictionary/service/src/main/java/datawave/microservice/dictionary/data/DataDictionaryImpl.java +++ b/microservices/services/dictionary/service/src/main/java/datawave/microservice/dictionary/data/DataDictionaryImpl.java @@ -32,14 +32,14 @@ public class DataDictionaryImpl implements DataDictionary { - private final MarkingFunctions markingFunctions; + private final MarkingFunctions markingFunctions; private final ResponseObjectFactory responseObjectFactory; private final MetadataHelperFactory metadataHelperFactory; private final MetadataDescriptionsHelperFactory metadataDescriptionsHelperFactory; private Map normalizationMap = Maps.newHashMap(); private String dataDictionarySystem = ""; - public DataDictionaryImpl(MarkingFunctions markingFunctions, + public DataDictionaryImpl(MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, MetadataHelperFactory metadataHelperFactory, MetadataDescriptionsHelperFactory metadataDescriptionsHelperFactory) { this.markingFunctions = markingFunctions; diff --git a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java index a4b5020af4b..45f62c0b532 100644 --- a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java +++ b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java @@ -39,13 +39,13 @@ public class DefaultMetadataFieldScanner { private static final String TIMESTAMP_FORMAT = "yyyyMMddHHmmss"; private static final Value EMPTY_VALUE = new Value(); - private final MarkingFunctions markingFunctions; + private final MarkingFunctions markingFunctions; private final ResponseObjectFactory responseObjectFactory; private final Map normalizationMap; private final Connection connectionConfig; private final int numThreads; - public DefaultMetadataFieldScanner(MarkingFunctions markingFunctions, + public DefaultMetadataFieldScanner(MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, Map normalizationMap, Connection connectionConfig, int numThreads) { this.markingFunctions = markingFunctions; diff --git a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java index 20d0e5ccc8f..8206e04b59c 100644 --- a/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java +++ b/microservices/services/dictionary/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java @@ -3,10 +3,8 @@ import static java.nio.charset.StandardCharsets.UTF_8; import java.util.Collections; -import java.util.Map; import java.util.Map.Entry; import java.util.Set; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import org.apache.accumulo.core.client.AccumuloClient; @@ -34,6 +32,7 @@ import datawave.data.ColumnFamilyConstants; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.microservice.dictionary.config.ResponseObjectFactory; import datawave.query.util.MetadataEntry; import datawave.security.util.ScannerHelper; @@ -48,14 +47,14 @@ public class MetadataDescriptionsHelper> { private static final Logger log = LoggerFactory.getLogger(MetadataDescriptionsHelper.class); - private final MarkingFunctions markingFunctions; + private final MarkingFunctions markingFunctions; private final ResponseObjectFactory responseObjectFactory; private String metadataTableName; private AccumuloClient accumuloClient; private Set fullUserAuths; - public MetadataDescriptionsHelper(MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { + public MetadataDescriptionsHelper(MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { this.markingFunctions = markingFunctions; this.responseObjectFactory = responseObjectFactory; } @@ -85,8 +84,7 @@ public SetMultimap getDescriptions(Set ingestTypeFil return descs; } - public SetMultimap getFieldDescriptions(Set ingestTypeFilter) - throws TableNotFoundException, ExecutionException, MarkingFunctions.Exception { + public SetMultimap getFieldDescriptions(Set ingestTypeFilter) throws TableNotFoundException, MarkingFunctions.Exception { SetMultimap descriptions = getDescriptions(ingestTypeFilter); SetMultimap fieldDescriptions = HashMultimap.create(); @@ -118,16 +116,21 @@ public void setDescription(MetadataEntry entry, DescriptionBase desc) setDescriptions(entry, Collections.singleton(desc)); } - public void setDescriptions(MetadataEntry entry, Set descs) - throws TableNotFoundException, MutationsRejectedException, MarkingFunctions.Exception { + public void setDescriptions(MetadataEntry entry, Set descs) throws TableNotFoundException, MutationsRejectedException { BatchWriter bw = null; try { BatchWriterConfig bwConfig = new BatchWriterConfig().setMaxMemory(10000L).setMaxLatency(100L, TimeUnit.MILLISECONDS).setMaxWriteThreads(1); bw = accumuloClient.createBatchWriter(metadataTableName, bwConfig); Mutation m = new Mutation(entry.getFieldName()); for (DescriptionBase desc : descs) { - m.put(ColumnFamilyConstants.COLF_DESC, new Text(entry.getDatatype()), markingFunctions.translateToColumnVisibility(desc.getMarkings()), - new Value(desc.getDescription().getBytes(UTF_8))); + ColumnVisibility cv = null; + if (desc.getMarkings() != null) { + cv = desc.getMarkings().toColumnVisibility(); + } + if (null == cv || cv.getExpression().length == 0) { + throw new IllegalArgumentException("ColumnVisibility was null or empty, unable to create Accumulo mutation"); + } + m.put(ColumnFamilyConstants.COLF_DESC, new Text(entry.getDatatype()), cv, new Value(desc.getDescription().getBytes(UTF_8))); } bw.addMutation(m); } finally { @@ -153,14 +156,20 @@ public void setDescriptions(MetadataEntry entry, Set * @throws MutationsRejectedException * if writing the update to Accumulo fails */ - public void removeDescription(MetadataEntry entry, DescriptionBase desc) - throws TableNotFoundException, MutationsRejectedException, MarkingFunctions.Exception { + public void removeDescription(MetadataEntry entry, DescriptionBase desc) throws TableNotFoundException, MutationsRejectedException { BatchWriter bw = null; try { BatchWriterConfig bwConfig = new BatchWriterConfig().setMaxMemory(10000L).setMaxLatency(100L, TimeUnit.MILLISECONDS).setMaxWriteThreads(1); bw = accumuloClient.createBatchWriter(metadataTableName, bwConfig); Mutation m = new Mutation(entry.getFieldName()); - m.putDelete(ColumnFamilyConstants.COLF_DESC, new Text(entry.getDatatype()), this.markingFunctions.translateToColumnVisibility(desc.getMarkings())); + ColumnVisibility cv = null; + if (desc.getMarkings() != null) { + cv = desc.getMarkings().toColumnVisibility(); + } + if (null == cv || cv.getExpression().length == 0) { + throw new IllegalArgumentException("ColumnVisibility was null or empty, unable to create Accumulo mutation"); + } + m.putDelete(ColumnFamilyConstants.COLF_DESC, new Text(entry.getDatatype()), cv); bw.addMutation(m); } finally { if (null != bw) { @@ -199,11 +208,7 @@ protected SetMultimap loadDescriptions() throws TableNotFoun } - private Map getMarkings(Key k) throws MarkingFunctions.Exception { - return getMarkings(k.getColumnVisibilityParsed()); - } - - private Map getMarkings(ColumnVisibility visibility) throws MarkingFunctions.Exception { - return markingFunctions.translateFromColumnVisibility(visibility); + private Markings getMarkings(Key k) throws MarkingFunctions.Exception { + return markingFunctions.translateFromColumnVisibility(k.getColumnVisibilityParsed()); } } diff --git a/microservices/services/dictionary/service/src/test/java/datawave/microservice/dictionary/DataDictionaryControllerTest.java b/microservices/services/dictionary/service/src/test/java/datawave/microservice/dictionary/DataDictionaryControllerTest.java index 1bc65a8e7cf..a8ff96dfa1b 100644 --- a/microservices/services/dictionary/service/src/test/java/datawave/microservice/dictionary/DataDictionaryControllerTest.java +++ b/microservices/services/dictionary/service/src/test/java/datawave/microservice/dictionary/DataDictionaryControllerTest.java @@ -2,8 +2,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; -import java.util.AbstractMap; -import java.util.HashMap; import java.util.Map; import org.apache.accumulo.core.client.AccumuloClient; @@ -32,7 +30,7 @@ import datawave.accumulo.inmemory.InMemoryAccumuloClient; import datawave.accumulo.inmemory.InMemoryInstance; -import datawave.marking.MarkingFunctions; +import datawave.marking.AccessExpressionMarkings; import datawave.microservice.ControllerIT; import datawave.microservice.dictionary.config.DataDictionaryProperties; import datawave.webservice.dictionary.data.DefaultDataDictionary; @@ -129,11 +127,9 @@ public void testPostDescriptions() { .build(); // @formatter:on - HashMap markings = new HashMap<>(); - markings.put(MarkingFunctions.Default.COLUMN_VISIBILITY, "USER|ADMIN"); Multimap,DefaultDescription> descriptions = HashMultimap.create(); - descriptions.put(new AbstractMap.SimpleEntry<>("fooField", "fooType"), new DefaultDescription("my foo field", markings)); - descriptions.put(new AbstractMap.SimpleEntry<>("barField", "barType"), new DefaultDescription("my bar field", markings)); + descriptions.put(Map.entry("fooField", "fooType"), new DefaultDescription("my foo field", AccessExpressionMarkings.create("USER"))); + descriptions.put(Map.entry("barField", "barType"), new DefaultDescription("my bar field", AccessExpressionMarkings.create("ADMIN"))); DefaultFields postBody = new DefaultFields(descriptions); MultiValueMap additionalHeaders = new LinkedMultiValueMap<>(); additionalHeaders.set(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE); diff --git a/microservices/services/dictionary/service/src/test/java/datawave/microservice/dictionary/data/DataDictionaryImplTest.java b/microservices/services/dictionary/service/src/test/java/datawave/microservice/dictionary/data/DataDictionaryImplTest.java index 7bc49d136ae..20254d103ce 100644 --- a/microservices/services/dictionary/service/src/test/java/datawave/microservice/dictionary/data/DataDictionaryImplTest.java +++ b/microservices/services/dictionary/service/src/test/java/datawave/microservice/dictionary/data/DataDictionaryImplTest.java @@ -28,6 +28,7 @@ import com.google.common.collect.Multimap; import com.google.common.collect.SetMultimap; +import datawave.marking.AccessExpressionMarkings; import datawave.marking.MarkingFunctions; import datawave.microservice.Connection; import datawave.microservice.dictionary.config.ResponseObjectFactory; @@ -59,7 +60,7 @@ public class DataDictionaryImplTest { private AccumuloClient accumuloClient; @Mock - private MarkingFunctions markingFunctions; + private MarkingFunctions markingFunctions; @Mock private ResponseObjectFactory responseObjectFactory; @@ -88,11 +89,8 @@ public void setUp() { @Test public void whenSettingDescription_givenSingleDefaultDescription_shouldSetDescription() throws Exception { - Map markings = new HashMap<>(); - markings.put("columnVisibility", "PRIVATE"); - DefaultDescription description = new DefaultDescription(); - description.setMarkings(markings); + description.setMarkings(AccessExpressionMarkings.create("PRIVATE")); description.setDescription("my ultra cool description"); // Ensure no alias will be found. @@ -110,11 +108,8 @@ public void whenSettingDescription_givenSingleDefaultDescription_shouldSetDescri @Test public void whenSettingDescription_givenDefaultDictionaryField_shouldSetDescriptionFromProperties() throws Exception { - Map markings = new HashMap<>(); - markings.put("columnVisibility", "PRIVATE"); - DefaultDescription description = new DefaultDescription(); - description.setMarkings(markings); + description.setMarkings(AccessExpressionMarkings.create("PRIVATE")); description.setDescription("my ultra cool description"); Set descriptions = Collections.singleton(description); @@ -138,11 +133,8 @@ public void whenSettingDescription_givenDefaultDictionaryField_shouldSetDescript @Test public void whenSettingDescription_givenNoAlias_shouldSetDescriptionWithOriginalFieldName() throws Exception { - Map markings = new HashMap<>(); - markings.put("columnVisibility", "PRIVATE"); - DefaultDescription description = new DefaultDescription(); - description.setMarkings(markings); + description.setMarkings(AccessExpressionMarkings.create("PRIVATE")); description.setDescription("my ultra cool description"); Set descriptions = Collections.singleton(description); @@ -161,11 +153,8 @@ public void whenSettingDescription_givenNoAlias_shouldSetDescriptionWithOriginal @Test public void whenSettingDescription_givenAlias_shouldSetDescriptionWithAlias() throws Exception { - Map markings = new HashMap<>(); - markings.put("columnVisibility", "PRIVATE"); - DefaultDescription description = new DefaultDescription(); - description.setMarkings(markings); + description.setMarkings(AccessExpressionMarkings.create("PRIVATE")); description.setDescription("my ultra cool description"); Set descriptions = Collections.singleton(description); @@ -294,11 +283,8 @@ public void whenRetrievingDescriptionsWithFieldNameAndDatatype_givenAlias_should @Test public void whenDeletingDescription_givenNoAlias_shouldDeleteDescriptionWithOriginalFieldName() throws Exception { - Map markings = new HashMap<>(); - markings.put("columnVisibility", "PRIVATE"); - DefaultDescription description = new DefaultDescription(); - description.setMarkings(markings); + description.setMarkings(AccessExpressionMarkings.create("PRIVATE")); description.setDescription("my ultra cool description"); // Ensure no alias will be found. @@ -315,11 +301,8 @@ public void whenDeletingDescription_givenNoAlias_shouldDeleteDescriptionWithOrig @Test public void whenDeletingDescription_givenAlias_shouldDeleteDescriptionWithAlias() throws Exception { - Map markings = new HashMap<>(); - markings.put("columnVisibility", "PRIVATE"); - DefaultDescription description = new DefaultDescription(); - description.setMarkings(markings); + description.setMarkings(AccessExpressionMarkings.create("PRIVATE")); description.setDescription("my ultra cool description"); // Ensure an alias will be found. diff --git a/microservices/services/dictionary/service/src/test/java/datawave/microservice/metadata/DefaultMetadataFieldScannerTest.java b/microservices/services/dictionary/service/src/test/java/datawave/microservice/metadata/DefaultMetadataFieldScannerTest.java index 894f896b8a8..e6c1fffe1b6 100644 --- a/microservices/services/dictionary/service/src/test/java/datawave/microservice/metadata/DefaultMetadataFieldScannerTest.java +++ b/microservices/services/dictionary/service/src/test/java/datawave/microservice/metadata/DefaultMetadataFieldScannerTest.java @@ -31,6 +31,7 @@ import datawave.accumulo.inmemory.InMemoryAccumuloClient; import datawave.accumulo.inmemory.InMemoryInstance; import datawave.data.ColumnFamilyConstants; +import datawave.marking.AccessExpressionMarkings; import datawave.marking.MarkingFunctions; import datawave.microservice.Connection; import datawave.microservice.dictionary.config.ResponseObjectFactory; @@ -50,7 +51,7 @@ public class DefaultMetadataFieldScannerTest { private static final String[] AUTH = {"PRIVATE"}; private static final Set AUTHS = Collections.singleton(new Authorizations(AUTH)); - private static final ResponseObjectFactory RESPONSE_OBJECT_FACTORY = new ResponseObjectFactory() { + private static final ResponseObjectFactory RESPONSE_OBJECT_FACTORY = new ResponseObjectFactory<>() { @Override public DefaultDataDictionary getDataDictionary() { return null; @@ -276,11 +277,7 @@ private void populateMetadataTable() throws TableNotFoundException, MutationsRej private DefaultDescription createDescription(String descriptionText) { DefaultDescription description = new DefaultDescription(); description.setDescription(descriptionText); - - Map markings = new HashMap<>(); - markings.put("columnVisibility", "PRIVATE}"); - description.setMarkings(markings); - + description.setMarkings(AccessExpressionMarkings.create("PRIVATE")); return description; } } diff --git a/microservices/services/query-metric/api/pom.xml b/microservices/services/query-metric/api/pom.xml index c393a690205..20e522afad7 100644 --- a/microservices/services/query-metric/api/pom.xml +++ b/microservices/services/query-metric/api/pom.xml @@ -21,7 +21,7 @@ ${basedir}/spotbugs-exclude.xml 3.20.0 1.9 - 7.33.1 + 7.40.0-SNAPSHOT 1.0.1 2.3.6 31.1-jre @@ -63,12 +63,6 @@ gov.nsa.datawave.core accumulo-utils ${version.datawave} - - - * - * - - gov.nsa.datawave.core diff --git a/microservices/services/query-metric/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java b/microservices/services/query-metric/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java index f4d2c88f678..2a63aa9307c 100644 --- a/microservices/services/query-metric/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java +++ b/microservices/services/query-metric/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java @@ -24,6 +24,7 @@ import javax.xml.bind.annotation.XmlElementWrapper; import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; +import org.apache.accumulo.access.AccessExpression; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.builder.CompareToBuilder; import org.apache.commons.lang3.builder.EqualsBuilder; @@ -34,7 +35,8 @@ import com.fasterxml.jackson.annotation.JsonIgnore; -import datawave.marking.MarkingFunctions; +import datawave.marking.AccessExpressionMarkings; +import datawave.marking.Markings; import datawave.microservice.query.Query; import datawave.microservice.query.QueryImpl.Parameter; import datawave.webservice.query.exception.QueryException; @@ -1165,21 +1167,21 @@ public void setPredictions(Set predictions) { } @Override - public void setMarkings(Map markings) { + public void setMarkings(Markings markings) { if (markings == null || markings.isEmpty()) { this.columnVisibility = null; } else { - this.columnVisibility = markings.get(MarkingFunctions.Default.COLUMN_VISIBILITY); + AccessExpression ae = markings.toAccessExpression(); + this.columnVisibility = ae.getExpression(); } } @Override - public Map getMarkings() { - Map markings = new HashMap<>(); + public Markings getMarkings() { if (this.columnVisibility != null) { - markings.put(MarkingFunctions.Default.COLUMN_VISIBILITY, this.columnVisibility); + return AccessExpressionMarkings.builder().columnVisibility(this.columnVisibility).build(); } - return markings; + return null; } public Schema getSchemaInstance() { diff --git a/microservices/services/query-metric/api/src/test/java/datawave/microservice/querymetric/QueryMetricTest.java b/microservices/services/query-metric/api/src/test/java/datawave/microservice/querymetric/QueryMetricTest.java index 5015edee51b..fe210ecd195 100644 --- a/microservices/services/query-metric/api/src/test/java/datawave/microservice/querymetric/QueryMetricTest.java +++ b/microservices/services/query-metric/api/src/test/java/datawave/microservice/querymetric/QueryMetricTest.java @@ -14,9 +14,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Date; -import java.util.HashMap; import java.util.List; -import java.util.Map; import javax.xml.bind.JAXBContext; import javax.xml.bind.Marshaller; @@ -28,7 +26,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.module.jaxb.JaxbAnnotationModule; -import datawave.marking.MarkingFunctions; +import datawave.marking.AccessExpressionMarkings; import datawave.microservice.querymetric.BaseQueryMetric.Lifecycle; import datawave.microservice.querymetric.BaseQueryMetric.PageMetric; import datawave.microservice.querymetric.BaseQueryMetric.Prediction; @@ -43,7 +41,6 @@ public class QueryMetricTest { private static QueryMetric queryMetric = null; - private static Map markings = null; private static List negativeSelectors = null; private static ArrayList pageTimes = null; private static List positiveSelectors = null; @@ -53,9 +50,7 @@ public class QueryMetricTest { @BeforeAll public static void setup() { queryMetric = new QueryMetric(); - markings = new HashMap<>(); - markings.put(MarkingFunctions.Default.COLUMN_VISIBILITY, "PUBLIC"); - queryMetric.setMarkings(markings); + queryMetric.setMarkings(AccessExpressionMarkings.builder().columnVisibility("PUBLIC").build()); negativeSelectors = new ArrayList<>(); negativeSelectors.add("negativeSelector1"); positiveSelectors = new ArrayList<>(); @@ -104,7 +99,7 @@ public void testSettersGetters() { queryMetric.setHost("host"); queryMetric.setLastUpdated(d); queryMetric.setLifecycle(Lifecycle.INITIALIZED); - queryMetric.setMarkings(markings); + queryMetric.setMarkings(AccessExpressionMarkings.builder().columnVisibility("PUBLIC").build()); queryMetric.setNegativeSelectors(negativeSelectors); queryMetric.setNumUpdates(0); queryMetric.setPageTimes(pageTimes); @@ -131,7 +126,7 @@ public void testSettersGetters() { assertEquals("host", queryMetric.getHost()); assertEquals(d, queryMetric.getLastUpdated()); assertEquals(Lifecycle.INITIALIZED, queryMetric.getLifecycle()); - assertEquals("PUBLIC", queryMetric.getMarkings().get(MarkingFunctions.Default.COLUMN_VISIBILITY)); + assertEquals("PUBLIC", queryMetric.getColumnVisibility()); assertEquals("negativeSelector1", queryMetric.getNegativeSelectors().get(0)); assertEquals(1, queryMetric.getNumPages()); assertEquals(0, queryMetric.getNumResults()); @@ -234,7 +229,7 @@ public void testVersionSerialization() throws Exception { qm.setHost("host"); qm.setLastUpdated(d); qm.setLifecycle(BaseQueryMetric.Lifecycle.INITIALIZED); - qm.setMarkings(markings); + qm.setMarkings(AccessExpressionMarkings.builder().columnVisibility("PUBLIC").build()); qm.setNegativeSelectors(negativeSelectors); qm.setNumUpdates(0); qm.setPageTimes(pageTimes); diff --git a/microservices/services/query-metric/service/pom.xml b/microservices/services/query-metric/service/pom.xml index 4623e1a5b89..e203023bed2 100644 --- a/microservices/services/query-metric/service/pom.xml +++ b/microservices/services/query-metric/service/pom.xml @@ -24,14 +24,14 @@ 1.1.1 3.3 3.20.0 - 7.33.1 - 4.0.1 + 7.40.0-SNAPSHOT + 4.0.2-SNAPSHOT 4.0.2 4.0.2 4.0.4 1.0.1 - 4.1.3 - 4.0.7 + 4.1.4-SNAPSHOT + 4.0.8-SNAPSHOT 3.0.4 3.0.5 31.1-jre @@ -135,12 +135,6 @@ gov.nsa.datawave.core accumulo-utils ${version.datawave} - - - * - * - - gov.nsa.datawave.core diff --git a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java index 158e3d2e7f7..79ee9eafb16 100644 --- a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java +++ b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java @@ -87,7 +87,7 @@ public class QueryMetricOperations { private QueryGeometryHandler geometryHandler; private CacheManager cacheManager; private Cache incomingQueryMetricsCache; - private MarkingFunctions markingFunctions; + private MarkingFunctions markingFunctions; private QueryMetricResponseFactory queryMetricResponseFactory; private MergeLockLifecycleListener mergeLock; private Correlator correlator; @@ -141,7 +141,7 @@ enum DEFAULT_DATETIME { */ @Autowired public QueryMetricOperations(@Qualifier("queryMetricCacheManager") CacheManager cacheManager, ShardTableQueryMetricHandler handler, - QueryGeometryHandler geometryHandler, MarkingFunctions markingFunctions, QueryMetricResponseFactory queryMetricResponseFactory, + QueryGeometryHandler geometryHandler, MarkingFunctions markingFunctions, QueryMetricResponseFactory queryMetricResponseFactory, MergeLockLifecycleListener mergeLock, Correlator correlator, MetricUpdateEntryProcessorFactory entryProcessorFactory, QueryMetricOperationsStats stats, QueryMetricClient queryMetricClient, DnUtils dnUtils) { this.handler = handler; @@ -478,11 +478,14 @@ private boolean isMetricsAdministrator(DatawaveUserDetails currentUser) { private boolean canViewMetric(DatawaveUserDetails currentUser, BaseQueryMetric metric) throws Exception { boolean userCanViewMetric = true; - ColumnVisibility columnVisibility = this.markingFunctions.translateToColumnVisibility(metric.getMarkings()); + ColumnVisibility columnVisibility = null; + if (null != metric.getMarkings()) { + columnVisibility = metric.getMarkings().toColumnVisibility(); + } for (DatawaveUser user : currentUser.getProxiedUsers()) { Authorizations authorizations = new Authorizations(user.getAuths().toArray(new String[0])); VisibilityEvaluator visibilityEvaluator = new VisibilityEvaluator(authorizations); - if (visibilityEvaluator.evaluate(columnVisibility) == false) { + if (!visibilityEvaluator.evaluate(columnVisibility)) { userCanViewMetric = false; break; } diff --git a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricHandlerConfiguration.java b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricHandlerConfiguration.java index a11c1ea1619..c7606028280 100644 --- a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricHandlerConfiguration.java +++ b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricHandlerConfiguration.java @@ -99,7 +99,7 @@ QueryMetricFactory queryMetricFactory() { @ConditionalOnMissingBean public ShardTableQueryMetricHandler shardTableQueryMetricHandler(QueryMetricHandlerProperties queryMetricHandlerProperties, AccumuloConnectionFactory connectionFactory, QueryMetricQueryLogicFactory logicFactory, QueryMetricFactory metricFactory, - MarkingFunctions markingFunctions, QueryMetricCombiner queryMetricCombiner, LuceneToJexlQueryParser luceneToJexlQueryParser, + MarkingFunctions markingFunctions, QueryMetricCombiner queryMetricCombiner, LuceneToJexlQueryParser luceneToJexlQueryParser, ResponseObjectFactory responseObjectFactory, WebClient.Builder webClientBuilder, @Autowired(required = false) JWTTokenHandler jwtTokenHandler, DnUtils dnUtils, QueryMetricResponseFactory queryMetricResponseFactory) { ShardTableQueryMetricHandler handler; diff --git a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/factory/QueryMetricQueryLogicFactory.java b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/factory/QueryMetricQueryLogicFactory.java index 68f8038704b..63ce5a661df 100644 --- a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/factory/QueryMetricQueryLogicFactory.java +++ b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/factory/QueryMetricQueryLogicFactory.java @@ -16,14 +16,14 @@ @Component public class QueryMetricQueryLogicFactory implements FactoryBean { - private MarkingFunctions markingFunctions; + private MarkingFunctions markingFunctions; private ResponseObjectFactory responseObjectFactory; private QueryMetricHandlerProperties queryMetricHandlerProperties; private DateIndexHelperFactory dateIndexHelperFactory; private MetadataHelperFactory metadataHelperFactory; @Autowired - public QueryMetricQueryLogicFactory(MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, + public QueryMetricQueryLogicFactory(MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, DateIndexHelperFactory dateIndexHelperFactory, MetadataHelperFactory metadataHelperFactory, QueryMetricHandlerProperties queryMetricHandlerProperties) { this.markingFunctions = markingFunctions; diff --git a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/LocalShardTableQueryMetricHandler.java b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/LocalShardTableQueryMetricHandler.java index 90b776d72bd..1a159f89789 100644 --- a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/LocalShardTableQueryMetricHandler.java +++ b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/LocalShardTableQueryMetricHandler.java @@ -48,7 +48,7 @@ public class LocalShardTableQueryMetricHandler extend protected ExecutorService executorService; public LocalShardTableQueryMetricHandler(QueryMetricHandlerProperties queryMetricHandlerProperties, AccumuloConnectionFactory connectionFactory, - QueryMetricQueryLogicFactory logicFactory, QueryMetricFactory metricFactory, MarkingFunctions markingFunctions, + QueryMetricQueryLogicFactory logicFactory, QueryMetricFactory metricFactory, MarkingFunctions markingFunctions, QueryMetricCombiner queryMetricCombiner, LuceneToJexlQueryParser luceneToJexlQueryParser, DnUtils dnUtils) { super(queryMetricHandlerProperties, connectionFactory, logicFactory, metricFactory, markingFunctions, queryMetricCombiner, luceneToJexlQueryParser, dnUtils); diff --git a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/RemoteShardTableQueryMetricHandler.java b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/RemoteShardTableQueryMetricHandler.java index 81eae0285ce..8381fdde6df 100644 --- a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/RemoteShardTableQueryMetricHandler.java +++ b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/RemoteShardTableQueryMetricHandler.java @@ -39,7 +39,7 @@ public class RemoteShardTableQueryMetricHandler exten private final JWTTokenHandler jwtTokenHandler; public RemoteShardTableQueryMetricHandler(QueryMetricHandlerProperties queryMetricHandlerProperties, AccumuloConnectionFactory connectionFactory, - QueryMetricQueryLogicFactory logicFactory, QueryMetricFactory metricFactory, MarkingFunctions markingFunctions, + QueryMetricQueryLogicFactory logicFactory, QueryMetricFactory metricFactory, MarkingFunctions markingFunctions, QueryMetricCombiner queryMetricCombiner, LuceneToJexlQueryParser luceneToJexlQueryParser, ResponseObjectFactory responseObjectFactory, WebClient.Builder webClientBuilder, JWTTokenHandler jwtTokenHandler, DnUtils dnUtils) { super(queryMetricHandlerProperties, connectionFactory, logicFactory, metricFactory, markingFunctions, queryMetricCombiner, luceneToJexlQueryParser, diff --git a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java index 351ba82ad4b..d5057e0dc53 100644 --- a/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java +++ b/microservices/services/query-metric/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java @@ -63,6 +63,7 @@ import datawave.ingest.protobuf.Uid; import datawave.ingest.table.config.TableConfigHelper; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.microservice.authorization.user.DatawaveUserDetails; import datawave.microservice.query.Query; import datawave.microservice.query.QueryImpl; @@ -106,13 +107,13 @@ public abstract class ShardTableQueryMetricHandler ex protected QueryMetricFactory metricFactory; protected UIDBuilder uidBuilder = UID.builder(); protected QueryMetricCombiner queryMetricCombiner; - protected MarkingFunctions markingFunctions; + protected MarkingFunctions markingFunctions; protected DnUtils dnUtils; // this lock is necessary for when there is an error condition and the accumuloRecordWriter needs to be replaced protected ReentrantReadWriteLock accumuloRecordWriterLock = new ReentrantReadWriteLock(); public ShardTableQueryMetricHandler(QueryMetricHandlerProperties queryMetricHandlerProperties, AccumuloConnectionFactory connectionFactory, - QueryMetricQueryLogicFactory logicFactory, QueryMetricFactory metricFactory, MarkingFunctions markingFunctions, + QueryMetricQueryLogicFactory logicFactory, QueryMetricFactory metricFactory, MarkingFunctions markingFunctions, QueryMetricCombiner queryMetricCombiner, LuceneToJexlQueryParser luceneToJexlQueryParser, DnUtils dnUtils) { super(luceneToJexlQueryParser); this.queryMetricHandlerProperties = queryMetricHandlerProperties; @@ -365,14 +366,9 @@ protected Multimap getEntries(ContentIndexingColumnBasedHan event.setDataType(type); event.setTimestamp(updatedQueryMetric.getCreateDate().getTime()); // get markings from metric, otherwise use the default markings - Map markings = updatedQueryMetric.getMarkings(); + Markings markings = updatedQueryMetric.getMarkings(); if (markings != null && !markings.isEmpty()) { - try { - event.setVisibility(this.markingFunctions.translateToColumnVisibility(updatedQueryMetric.getMarkings())); - } catch (MarkingFunctions.Exception e) { - log.error(e.getMessage(), e); - event.setVisibility(this.queryMetricHandlerProperties.getDefaultMetricVisibility()); - } + event.setVisibility(markings.toColumnVisibility()); } else { event.setVisibility(this.queryMetricHandlerProperties.getDefaultMetricVisibility()); } diff --git a/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/MetadataCachingTest.java b/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/MetadataCachingTest.java index cebdf520ada..d4b12b59e85 100644 --- a/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/MetadataCachingTest.java +++ b/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/MetadataCachingTest.java @@ -11,8 +11,8 @@ import java.util.stream.Collectors; import org.apache.accumulo.core.security.Authorizations; -import org.junit.Assert; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -79,6 +79,7 @@ public void VerifyMetadataMethodsCacheable() throws Exception { metadataHelper.getDatatypes(Collections.singleton("querymetrics")); metadataHelper.getFieldsForDatatype(LcNoDiacriticsType.class); metadataHelper.getFieldsToDatatypes(Collections.singleton("querymetrics")); + metadataHelper.getHiddenFields(Collections.singleton("querymetrics")); metadataHelper.getContentFields(Collections.singleton("querymetrics")); metadataHelper.getExpansionFields(Collections.singleton("querymetrics")); metadataHelper.getQueryModel(queryMetricHandlerProperties.getMetadataTableName(), "querymetrics"); @@ -110,7 +111,7 @@ public void VerifyMetadataMethodsCacheable() throws Exception { expectedCacheNames.addAll(getCacheNamesForCacheableMethods(AllFieldMetadataHelper.class)); expectedCacheNames.addAll(getCacheNamesForCacheableMethods(DateIndexHelper.class)); - Assert.assertEquals(expectedCacheNames, cacheNames); + Assertions.assertEquals(expectedCacheNames, cacheNames); } private List getCacheNamesForCacheableMethods(Class clazz) { diff --git a/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/NonWebApplicationMessagingTest.java b/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/NonWebApplicationMessagingTest.java index 7d82ff5d9f5..4ee5321ae9d 100644 --- a/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/NonWebApplicationMessagingTest.java +++ b/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/NonWebApplicationMessagingTest.java @@ -23,7 +23,7 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit.jupiter.SpringExtension; -import datawave.marking.MarkingFunctions; +import datawave.marking.ColumnVisibilitySecurityMarking; import datawave.microservice.querymetric.config.QueryMetricTransportType; import datawave.microservice.querymetric.config.TimelyProperties; import datawave.microservice.querymetric.function.QueryMetricSupplier; @@ -53,7 +53,7 @@ public class NonWebApplicationMessagingTest { @BeforeEach public void setup() { this.metricMarkings = new HashMap<>(); - this.metricMarkings.put(MarkingFunctions.Default.COLUMN_VISIBILITY, "A&C"); + this.metricMarkings.put(ColumnVisibilitySecurityMarking.VISIBILITY_MARKING, "A&C"); this.storedMetricUpdates.clear(); } diff --git a/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java b/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java index a61f8dd3cf8..1e422b8e02b 100644 --- a/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java +++ b/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java @@ -1,5 +1,6 @@ package datawave.microservice.querymetric; +import static datawave.marking.AccessExpressionMarkings.ACCESS; import static datawave.microservice.querymetric.config.HazelcastMetricCacheConfiguration.INCOMING_METRICS; import static datawave.security.authorization.DatawaveUser.UserType.USER; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -12,13 +13,13 @@ import java.util.Collection; import java.util.Collections; import java.util.Date; -import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.BatchScanner; import org.apache.accumulo.core.client.BatchWriter; @@ -65,7 +66,8 @@ import datawave.core.common.connection.AccumuloConnectionFactory; import datawave.ingest.protobuf.Uid; -import datawave.marking.MarkingFunctions; +import datawave.marking.AccessExpressionMarkings; +import datawave.marking.Markings; import datawave.microservice.authorization.preauth.ProxiedEntityX509Filter; import datawave.microservice.authorization.user.DatawaveUserDetails; import datawave.microservice.querymetric.config.QueryMetricClientProperties; @@ -144,14 +146,14 @@ public class QueryMetricTestBase { protected DatawaveUserDetails nonAdminUser; protected static boolean isHazelCast; protected static CacheManager staticCacheManager; - protected static Map metricMarkings; + protected static Markings metricMarkings; protected List tables; protected Collection auths; protected AccumuloClient accumuloClient; static { - metricMarkings = new HashMap<>(); - metricMarkings.put(MarkingFunctions.Default.COLUMN_VISIBILITY, "A&C"); + AccessExpression ae = ACCESS.newExpression("A&C"); + metricMarkings = AccessExpressionMarkings.builder().accessExpression(ae).build(); } @AfterAll diff --git a/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/config/AlternateQueryMetricConfiguration.java b/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/config/AlternateQueryMetricConfiguration.java index 3695bf7602e..eb856d51e77 100644 --- a/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/config/AlternateQueryMetricConfiguration.java +++ b/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/config/AlternateQueryMetricConfiguration.java @@ -48,7 +48,7 @@ public BaseQueryMetric createMetric(boolean populateVersionMap) { @Bean public ShardTableQueryMetricHandler shardTableQueryMetricHandler(QueryMetricHandlerProperties queryMetricHandlerProperties, AccumuloConnectionFactory connectionFactory, QueryMetricQueryLogicFactory logicFactory, QueryMetricFactory metricFactory, - MarkingFunctions markingFunctions, QueryMetricCombiner queryMetricCombiner, LuceneToJexlQueryParser luceneToJexlQueryParser, + MarkingFunctions markingFunctions, QueryMetricCombiner queryMetricCombiner, LuceneToJexlQueryParser luceneToJexlQueryParser, DnUtils dnUtils) { return new AlternateShardTableQueryMetricHandler(queryMetricHandlerProperties, connectionFactory, logicFactory, metricFactory, markingFunctions, queryMetricCombiner, luceneToJexlQueryParser, dnUtils); diff --git a/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/config/AlternateShardTableQueryMetricHandler.java b/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/config/AlternateShardTableQueryMetricHandler.java index ff985dc0bd8..8f71e434bf2 100644 --- a/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/config/AlternateShardTableQueryMetricHandler.java +++ b/microservices/services/query-metric/service/src/test/java/datawave/microservice/querymetric/config/AlternateShardTableQueryMetricHandler.java @@ -18,7 +18,7 @@ public class AlternateShardTableQueryMetricHandler extends LocalShardTableQueryMetricHandler { public AlternateShardTableQueryMetricHandler(QueryMetricHandlerProperties queryMetricHandlerProperties, AccumuloConnectionFactory connectionFactory, - QueryMetricQueryLogicFactory logicFactory, QueryMetricFactory metricFactory, MarkingFunctions markingFunctions, + QueryMetricQueryLogicFactory logicFactory, QueryMetricFactory metricFactory, MarkingFunctions markingFunctions, QueryMetricCombiner queryMetricCombiner, LuceneToJexlQueryParser luceneToJexlQueryParser, DnUtils dnUtils) { super(queryMetricHandlerProperties, connectionFactory, logicFactory, metricFactory, markingFunctions, queryMetricCombiner, luceneToJexlQueryParser, dnUtils); diff --git a/microservices/starters/audit/pom.xml b/microservices/starters/audit/pom.xml index 866c815f03a..749d0eb590d 100644 --- a/microservices/starters/audit/pom.xml +++ b/microservices/starters/audit/pom.xml @@ -25,10 +25,10 @@ 1.9.25.1 3.20.0 - 7.33.1 + 7.40.0-SNAPSHOT 4.0.2 4.0.2 - 4.0.7 + 4.0.8-SNAPSHOT 31.1-jre 2.2 diff --git a/microservices/starters/audit/src/main/java/datawave/microservice/audit/AuditClient.java b/microservices/starters/audit/src/main/java/datawave/microservice/audit/AuditClient.java index b0167b38a61..4709b2b64c8 100644 --- a/microservices/starters/audit/src/main/java/datawave/microservice/audit/AuditClient.java +++ b/microservices/starters/audit/src/main/java/datawave/microservice/audit/AuditClient.java @@ -165,7 +165,7 @@ protected Request(Builder b) throws IllegalArgumentException { params.set(AuditParameters.QUERY_AUDIT_TYPE, b.auditType.name()); } if (null != b.marking) { - params.set(AuditParameters.QUERY_SECURITY_MARKING_COLVIZ, b.marking.toColumnVisibilityString()); + params.set(AuditParameters.QUERY_SECURITY_MARKING_COLVIZ, b.marking.toAccessExpressionString()); } if (null != b.queryLogic) { params.set(AuditParameters.QUERY_LOGIC_CLASS, b.queryLogic); diff --git a/microservices/starters/datawave/pom.xml b/microservices/starters/datawave/pom.xml index 2dc04cc69e8..b6a59b07ced 100644 --- a/microservices/starters/datawave/pom.xml +++ b/microservices/starters/datawave/pom.xml @@ -26,7 +26,7 @@ 2.1.4-5792fed3-bulkv2 1.9.25.1 1.9.4 - 7.33.1 + 7.40.0-SNAPSHOT 4.0.2 3.0.0 4.0.2 @@ -90,12 +90,6 @@ gov.nsa.datawave.core accumulo-utils ${version.datawave} - - - ch.qos.logback - * - - gov.nsa.datawave.core diff --git a/microservices/starters/datawave/src/main/java/datawave/microservice/config/marking/MarkingConfig.java b/microservices/starters/datawave/src/main/java/datawave/microservice/config/marking/MarkingConfig.java index 0160a938824..0bbfa80d74c 100644 --- a/microservices/starters/datawave/src/main/java/datawave/microservice/config/marking/MarkingConfig.java +++ b/microservices/starters/datawave/src/main/java/datawave/microservice/config/marking/MarkingConfig.java @@ -24,7 +24,7 @@ public class MarkingConfig { @Bean @ConditionalOnMissingBean @ConditionalOnProperty(name = "datawave.defaults.MarkingFunctions.enabled", havingValue = "true", matchIfMissing = true) - public MarkingFunctions markingFunctions() { + public MarkingFunctions markingFunctions() { return new MarkingFunctions.Default(); } diff --git a/pom.xml b/pom.xml index aa849568694..722dbc9ce94 100644 --- a/pom.xml +++ b/pom.xml @@ -75,14 +75,14 @@ 1.11.0 5.9.0 5.9.0 - 4.0.0 - 4.0.0 - 4.0.0 - 4.0.5 - 1.0.0 + 4.0.2-SNAPSHOT + 4.0.2 + 4.0.2 + 4.0.7-SNAPSHOT + 1.0.1 3.0.0 - 1.0.0 - 4.0.7 + 1.0.1 + 4.1.4-SNAPSHOT 1.9.0 3.6.3 5.2.0 @@ -124,6 +124,7 @@ 2.20 2.20 2.25.4 + 1.18.36 7.5.0 1.6.0 1.2 @@ -1375,6 +1376,12 @@ + + org.projectlombok + lombok + ${version.lombok} + provided + org.wildfly wildfly-security diff --git a/warehouse/core/src/main/java/datawave/marking/MarkingFunctionsFactory.java b/warehouse/core/src/main/java/datawave/marking/MarkingFunctionsFactory.java index 256d44750a4..0f2fd62de79 100644 --- a/warehouse/core/src/main/java/datawave/marking/MarkingFunctionsFactory.java +++ b/warehouse/core/src/main/java/datawave/marking/MarkingFunctionsFactory.java @@ -21,13 +21,13 @@ public class MarkingFunctionsFactory { @Inject @SpringBean(refreshable = true) - private static MarkingFunctions applicationMarkingFunctions; + private static MarkingFunctions applicationMarkingFunctions; public static final Logger log = LoggerFactory.getLogger(MarkingFunctionsFactory.class); - private static MarkingFunctions markingFunctions; + private static MarkingFunctions markingFunctions; - public static synchronized MarkingFunctions createMarkingFunctions() { + public static synchronized MarkingFunctions createMarkingFunctions() { if (markingFunctions != null) return markingFunctions; diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/config/RawRecordContainerImpl.java b/warehouse/ingest-core/src/main/java/datawave/ingest/config/RawRecordContainerImpl.java index 96dea3dc999..b1fa5f63ac4 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/config/RawRecordContainerImpl.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/config/RawRecordContainerImpl.java @@ -15,8 +15,8 @@ import java.util.Set; import java.util.concurrent.ConcurrentSkipListSet; +import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.security.ColumnVisibility; -import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.ToStringBuilder; import org.apache.hadoop.conf.Configurable; @@ -36,7 +36,8 @@ import datawave.ingest.data.config.ConfigurationHelper; import datawave.ingest.data.config.ingest.IgnorableErrorHelperInterface; import datawave.ingest.protobuf.RawRecordContainer.Data; -import datawave.marking.MarkingFunctions; +import datawave.marking.AccessExpressionMarkings; +import datawave.marking.Markings; import datawave.util.CompositeTimestamp; public class RawRecordContainerImpl implements Writable, Configurable, RawRecordContainer { @@ -81,56 +82,41 @@ public class RawRecordContainerImpl implements Writable, Configurable, RawRecord private Map auxMap = null; // RawRecordContainer support - Map securityMarkings = null; + Markings securityMarkings = null; public RawRecordContainerImpl() { uidBuilder = UID.builder(); } @Override - public Map getSecurityMarkings() { + public Markings getSecurityMarkings() { return this.securityMarkings; } @Override - public void setSecurityMarkings(Map securityMarkings) { - this.securityMarkings = (securityMarkings == null ? null : new HashMap<>(securityMarkings)); + public void setSecurityMarkings(Markings securityMarkings) { + this.securityMarkings = securityMarkings; syncSecurityMarkingsToFields(); } - @Override - public void addSecurityMarking(String domain, String marking) { - if (null == securityMarkings) { - securityMarkings = new HashMap<>(); - } - securityMarkings.put(domain, marking); - syncSecurityMarkingsToFields(); - } - - @Override - public boolean hasSecurityMarking(String domain, String marking) { - return null != securityMarkings && securityMarkings.containsKey(domain) && StringUtils.equals(securityMarkings.get(domain), marking); - } - protected void syncSecurityMarkingsToFields() { if (securityMarkings != null) { - setVisibility(securityMarkings.get(MarkingFunctions.Default.COLUMN_VISIBILITY)); + AccessExpression ae = securityMarkings.toAccessExpression(); + if (ae != null) { + setVisibilityNoSync(ae.getExpression()); + } else { + setVisibilityNoSync(null); + } } else { - setVisibility((String) null); + setVisibilityNoSync(null); } } protected void syncFieldsToSecurityMarkings() { if (visibility != null) { - if (securityMarkings == null) { - securityMarkings = new HashMap<>(); - } - securityMarkings.put(MarkingFunctions.Default.COLUMN_VISIBILITY, new String(visibility.getExpression())); - - } else if (securityMarkings != null) { - securityMarkings.remove(MarkingFunctions.Default.COLUMN_VISIBILITY); - } - if (securityMarkings != null && securityMarkings.isEmpty()) { + String expr = new String(visibility.getExpression()); + securityMarkings = AccessExpressionMarkings.builder().columnVisibility(expr).build(); + } else { securityMarkings = null; } } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/RawRecordContainer.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/RawRecordContainer.java index 756f8bb26e7..1628282835f 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/RawRecordContainer.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/RawRecordContainer.java @@ -4,11 +4,11 @@ import java.io.IOException; import java.util.Collection; import java.util.Date; -import java.util.Map; import org.apache.accumulo.core.security.ColumnVisibility; import datawave.data.hash.UID; +import datawave.marking.Markings; import datawave.util.CompositeTimestamp; /** @@ -17,13 +17,9 @@ */ public interface RawRecordContainer { - Map getSecurityMarkings(); + Markings getSecurityMarkings(); - void setSecurityMarkings(Map securityMarkings); - - void addSecurityMarking(String domain, String marking); - - boolean hasSecurityMarking(String domain, String marking); + void setSecurityMarkings(Markings securityMarkings); UID getId(); diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/BaseNormalizedContent.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/BaseNormalizedContent.java index 4737e284ad4..3373d4e709d 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/BaseNormalizedContent.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/BaseNormalizedContent.java @@ -1,11 +1,10 @@ package datawave.ingest.data.config; -import java.util.Map; - import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; import datawave.data.type.Type; +import datawave.marking.Markings; /** * Base container class implementation of the NormalizedContentInterface. This class is utilized to retain the original and transformed content and labels for a @@ -22,7 +21,7 @@ public class BaseNormalizedContent implements NormalizedContentInterface, Clonea protected String _indexedFieldValue = null; /** The security markings for the field value pair. */ - protected Map _markings = null; + protected Markings _markings = null; /** The field processing error if any. */ protected Throwable error = null; @@ -40,7 +39,7 @@ public BaseNormalizedContent(String field, String value) { _indexedFieldValue = value; } - public BaseNormalizedContent(String field, String value, Map markings) { + public BaseNormalizedContent(String field, String value, Markings markings) { this(field, value); _markings = markings; } @@ -117,12 +116,12 @@ public void setEventFieldValue(String originalFieldValue) { } @Override - public Map getMarkings() { + public Markings getMarkings() { return _markings; } @Override - public void setMarkings(Map markings) { + public void setMarkings(Markings markings) { hashCode = null; this._markings = markings; } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/MarkingsHelper.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/MarkingsHelper.java index 1f61a7a1e8b..157f313817f 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/MarkingsHelper.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/MarkingsHelper.java @@ -8,7 +8,8 @@ import datawave.ingest.data.Type; import datawave.ingest.data.config.ingest.BaseIngestHelper; -import datawave.marking.MarkingFunctions; +import datawave.marking.AccessExpressionMarkings; +import datawave.marking.Markings; public interface MarkingsHelper { @@ -28,7 +29,7 @@ public interface MarkingsHelper { * * @return markings */ - Map getDefaultMarkings(); + Markings getDefaultMarkings(); /** * Returns the markings override for the field @@ -37,7 +38,7 @@ public interface MarkingsHelper { * the field name * @return markings */ - Map getFieldMarking(String fieldName); + Markings getFieldMarking(String fieldName); /** * Method to set the markings on a field @@ -48,12 +49,12 @@ public interface MarkingsHelper { void markField(NormalizedContentInterface field); /** - * No-op helper for default ColumnVisibility implementation. Should only be used for testing purposes. + * No-op helper for default AccessExpression implementation. Should only be used for testing purposes. */ class NoOp implements MarkingsHelper { - private Map> fieldMarkingMap = new HashMap<>(); - private Map defaultMarkings = null; + private Map> fieldMarkingMap = new HashMap<>(); + private Markings defaultMarkings = null; Configuration conf; Type dataType; @@ -67,15 +68,13 @@ public NoOp(Configuration conf, Type dataType) { private void initDefaultMarkings() { String marking = conf.get(dataType.typeName() + DEFAULT_MARKING); if (null != marking) { - defaultMarkings = new HashMap<>(); - defaultMarkings.put(MarkingFunctions.Default.COLUMN_VISIBILITY, marking); + defaultMarkings = AccessExpressionMarkings.builder().columnVisibility(marking).build(); } for (Entry property : conf) { if (property.getKey().startsWith(dataType.typeName()) && property.getKey().endsWith(FIELD_MARKING)) { String fieldName = null; if (null != (fieldName = BaseIngestHelper.getFieldName(dataType, property.getKey(), FIELD_MARKING))) { - Map fieldMarking = new HashMap<>(); - fieldMarking.put(MarkingFunctions.Default.COLUMN_VISIBILITY, property.getValue()); + Markings fieldMarking = AccessExpressionMarkings.builder().columnVisibility(property.getValue()).build(); fieldMarkingMap.put(fieldName, fieldMarking); } } @@ -83,12 +82,12 @@ private void initDefaultMarkings() { } @Override - public Map getDefaultMarkings() { + public Markings getDefaultMarkings() { return defaultMarkings; } @Override - public Map getFieldMarking(String fieldName) { + public Markings getFieldMarking(String fieldName) { return fieldMarkingMap.get(fieldName); } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/NormalizedContentInterface.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/NormalizedContentInterface.java index 86c5df3eef1..1061afacb99 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/NormalizedContentInterface.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/NormalizedContentInterface.java @@ -1,8 +1,7 @@ package datawave.ingest.data.config; -import java.util.Map; - import datawave.data.type.Type; +import datawave.marking.Markings; /** * Interface utilized to reference the original and transformed content and labels for a value pair ingested into accumulo. @@ -66,7 +65,7 @@ public interface NormalizedContentInterface extends Cloneable { * @param markings * the markings to set */ - void setMarkings(Map markings); + void setMarkings(Markings markings); /** * When we fail to process a field (e.g. normalize), then an error can be set which in the EventMapper results in the event being dropped from normal @@ -82,7 +81,7 @@ public interface NormalizedContentInterface extends Cloneable { * * @return The field specific markings, or null if the overall event markings are to be used. */ - Map getMarkings(); + Markings getMarkings(); /** * Get the processing error for this field if any. diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/NormalizedFieldAndValue.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/NormalizedFieldAndValue.java index f1adc065713..c7a5e67060d 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/NormalizedFieldAndValue.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/NormalizedFieldAndValue.java @@ -6,7 +6,6 @@ * will be used. */ import java.io.UnsupportedEncodingException; -import java.util.Map; import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; @@ -15,6 +14,7 @@ import datawave.ingest.config.IngestConfiguration; import datawave.ingest.config.IngestConfigurationFactory; import datawave.ingest.config.MimeDecoder; +import datawave.marking.Markings; public class NormalizedFieldAndValue extends BaseNormalizedContent implements GroupedNormalizedContentInterface { private boolean grouped = false; @@ -42,7 +42,7 @@ public NormalizedFieldAndValue(String field, String value) { super(field, value); } - public NormalizedFieldAndValue(String field, String value, Map markings) { + public NormalizedFieldAndValue(String field, String value, Markings markings) { super(field, value, markings); } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CompositeIngest.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CompositeIngest.java index a5fef654c62..6646037dc02 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CompositeIngest.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CompositeIngest.java @@ -30,6 +30,7 @@ import datawave.ingest.data.config.NormalizedContentInterface; import datawave.ingest.data.config.NormalizedFieldAndValue; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; /** * Similar to VirtualIngest virtual fields, but composite fields are not written in the event section of the shard table (only in the field index section) @@ -131,7 +132,7 @@ class CompositeFieldNormalizer { protected Map allowMissing = new HashMap<>(); private Set ignoreNormalizationForFields = new HashSet<>(); - protected MarkingFunctions markingFunctions; + protected MarkingFunctions markingFunctions; public void setup(Type type, Configuration config) { @@ -285,30 +286,6 @@ private String[] getConfPrefixes(String type) { return new String[] {str2, str1}; } - /** - * A helper routine to merge markings maps when merging fields of a NormalizedContentInterface - * - * @param markings1 - * a map of markings - * @param markings2 - * a different map of markings - * @return the merged markings - */ - protected Map mergeMarkings(Map markings1, Map markings2) { - if (markings2 != null) { - if (markings1 == null) { - markings1 = markings2; - } else { - try { - markings1 = markingFunctions.combine(markings1, markings2); - } catch (MarkingFunctions.Exception e) { - throw new RuntimeException("Unable to combine markings.", e); - } - } - } - return markings1; - } - public Multimap normalize(Multimap fields) { Multimap eventFields = HashMultimap.create(); for (Entry field : fields.entries()) { @@ -356,7 +333,7 @@ private void compilePatterns() { // Create the composite field from the event Fields members public void addCompositeFields(List compositeFields, Multimap eventFields, String compositeFieldName, String replacement, String[] grouping, GroupingPolicy groupingPolicy, boolean allowMissing, String[] fields, - int pos, StringBuilder originalValue, StringBuilder normalizedValue, Map markings, boolean isOverloadedField) { + int pos, StringBuilder originalValue, StringBuilder normalizedValue, Markings markings, boolean isOverloadedField) { String separator = (pos > 0) ? compositeSeparator.get(compositeFieldName) : ""; // append any constants that have been specified while (pos < fields.length && isConstant(fields[pos])) { @@ -405,9 +382,14 @@ public void addCompositeFields(List compositeFields, normalizedValue.append(separator); normalizedValue.append(value.getIndexedFieldValue()); if (pos + 1 < fields.length) { - addCompositeFields(compositeFields, eventFields, compositeFieldName.replace("*", replacement), replacement, - (grouping == null ? newGrouping : grouping), groupingPolicy, allowMissing, fields, pos + 1, originalValue, - normalizedValue, mergeMarkings(markings, value.getMarkings()), isOverloadedField); + try { + addCompositeFields(compositeFields, eventFields, compositeFieldName.replace("*", replacement), replacement, + (grouping == null ? newGrouping : grouping), groupingPolicy, allowMissing, fields, pos + 1, + originalValue, normalizedValue, markingFunctions.combine(markings, value.getMarkings()), + isOverloadedField); + } catch (MarkingFunctions.Exception e) { + throw new RuntimeException("Could not combine markings", e); + } } originalValue.setLength(oLen); normalizedValue.setLength(nLen); @@ -427,7 +409,7 @@ public void addCompositeFields(List compositeFields, continue; } } - if (!isOverloadedField || (isOverloadedField && pos == 0)) { + if (!isOverloadedField || pos == 0) { originalValue.append(separator); originalValue.append(value.getEventFieldValue()); } @@ -437,9 +419,13 @@ public void addCompositeFields(List compositeFields, } else { normalizedValue.append(value.getIndexedFieldValue()); } - addCompositeFields(compositeFields, eventFields, compositeFieldName, replacement, (grouping == null ? newGrouping : grouping), - groupingPolicy, allowMissing, fields, pos + 1, originalValue, normalizedValue, - mergeMarkings(markings, value.getMarkings()), isOverloadedField); + try { + addCompositeFields(compositeFields, eventFields, compositeFieldName, replacement, (grouping == null ? newGrouping : grouping), + groupingPolicy, allowMissing, fields, pos + 1, originalValue, normalizedValue, + markingFunctions.combine(markings, value.getMarkings()), isOverloadedField); + } catch (MarkingFunctions.Exception e) { + throw new RuntimeException("Could not combine markings", e); + } originalValue.setLength(oLen); normalizedValue.setLength(nLen); } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java index b24bb798f9f..e02a757db21 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java @@ -28,6 +28,7 @@ import datawave.ingest.data.config.NormalizedContentInterface; import datawave.ingest.data.config.NormalizedFieldAndValue; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; public interface VirtualIngest { @@ -126,7 +127,7 @@ class VirtualFieldNormalizer extends NoOpType { protected transient Map allowMissing = new HashMap<>(); private Set ignoreNormalizationForFields = new HashSet<>(); - private transient MarkingFunctions markingFunctions; + private transient MarkingFunctions markingFunctions; public void setup(Type type, String instance, Configuration config) { @@ -338,30 +339,6 @@ private String[] getConfPrefixes(String type, String instance) { } } - /** - * A helper routine to merge markings maps when merging fields of a NormalizedContentInterface - * - * @param markings1 - * a map of markings - * @param markings2 - * a different map of markings - * @return the merged markings - */ - protected Map mergeMarkings(Map markings1, Map markings2) { - if (markings2 != null) { - if (markings1 == null) { - markings1 = markings2; - } else { - try { - markings1 = markingFunctions.combine(markings1, markings2); - } catch (MarkingFunctions.Exception e) { - throw new RuntimeException("Unable to combine markings.", e); - } - } - } - return markings1; - } - /** * Create the normalized form of a map of fields, and add the virtual fields as configured above. * @@ -544,7 +521,7 @@ public String toString() { public void addVirtualFields(List virtualFields, Multimap eventFields, Map> groupings, String virtualFieldName, String replacement, VirtualFieldGrouping grouping, GroupingPolicy groupingPolicy, boolean allowMissing, String[] fields, int pos, String startSeparator, - String endSeparator, StringBuilder originalValue, StringBuilder normalizedValue, Map markings) { + String endSeparator, StringBuilder originalValue, StringBuilder normalizedValue, Markings markings) { String separator = ""; // append any constants that have been specified while (pos < fields.length && isConstant(fields[pos])) { @@ -599,10 +576,14 @@ public void addVirtualFields(List virtualFields, Mul normalizedValue.append(endSeparator); // recurse for the next virtual field definition segment - addVirtualFields(virtualFields, eventFields, groupings, virtualFieldName.replace("*", replacement), replacement, - (grouping == null ? newGrouping : grouping), groupingPolicy, allowMissing, fields, pos + 1, - this.defaultStartSeparator, this.defaultEndSeparator, originalValue, normalizedValue, - mergeMarkings(markings, value.getMarkings())); + try { + addVirtualFields(virtualFields, eventFields, groupings, virtualFieldName.replace("*", replacement), replacement, + (grouping == null ? newGrouping : grouping), groupingPolicy, allowMissing, fields, pos + 1, + this.defaultStartSeparator, this.defaultEndSeparator, originalValue, normalizedValue, + markingFunctions.combine(markings, value.getMarkings())); + } catch (MarkingFunctions.Exception e) { + throw new RuntimeException("Could not combine markings", e); + } // reset the values to the original length originalValue.setLength(oLen); @@ -638,9 +619,13 @@ public void addVirtualFields(List virtualFields, Mul normalizedValue.append(endSeparator); // recurse on the next virtual field segment - addVirtualFields(virtualFields, eventFields, groupings, virtualFieldName, replacement, (grouping == null ? newGrouping : grouping), - groupingPolicy, allowMissing, fields, pos + 1, this.defaultStartSeparator, this.defaultEndSeparator, originalValue, - normalizedValue, mergeMarkings(markings, value.getMarkings())); + try { + addVirtualFields(virtualFields, eventFields, groupings, virtualFieldName, replacement, (grouping == null ? newGrouping : grouping), + groupingPolicy, allowMissing, fields, pos + 1, this.defaultStartSeparator, this.defaultEndSeparator, originalValue, + normalizedValue, markingFunctions.combine(markings, value.getMarkings())); + } catch (MarkingFunctions.Exception e) { + throw new RuntimeException("Could not combine markings", e); + } // reset the values to the original length originalValue.setLength(oLen); diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/EventMapper.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/EventMapper.java index d0ccd56b113..89be13c7b63 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/EventMapper.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/EventMapper.java @@ -161,7 +161,7 @@ public class EventMapper extends StatsDE protected Set sequenceFileNames = new HashSet<>(); - protected MarkingFunctions markingFunctions; + protected MarkingFunctions markingFunctions; private boolean metricsEnabled = false; private MetricsService metricsService; diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomDataTypeHandler.java index 5ba56f69fda..103f61853c9 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomDataTypeHandler.java @@ -30,6 +30,7 @@ import datawave.ingest.mapreduce.job.writer.ContextWriter; import datawave.ingest.metadata.RawRecordMetadata; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.util.StringUtils; import datawave.util.TextUtil; @@ -63,7 +64,7 @@ public class AtomDataTypeHandler implements ExtendedDataT protected String[] fieldOverrides = null; protected HashMap> subCategories; protected String[] sCategories = null; - protected MarkingFunctions markingFunctions; + protected MarkingFunctions markingFunctions; protected Configuration conf; @@ -254,16 +255,11 @@ public Key createKey(String fieldName, String fieldValue, String columnQualifier * the value * @return the visibility */ - protected String getColumnQualifier(RawRecordContainer event, NormalizedContentInterface value) { ColumnVisibility visibility = event.getVisibility(); - if (value.getMarkings() != null && !value.getMarkings().isEmpty()) { - try { - visibility = markingFunctions.translateToColumnVisibility(value.getMarkings()); - } catch (MarkingFunctions.Exception e) { - throw new RuntimeException("Cannot convert record-level markings into a column visibility", e); - - } + Markings markings = value.getMarkings(); + if (markings != null && !markings.isEmpty()) { + visibility = markings.toColumnVisibility(); } return flatten(visibility); } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomErrorDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomErrorDataTypeHandler.java index aa8c731581c..9676e9a8fc2 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomErrorDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomErrorDataTypeHandler.java @@ -5,6 +5,7 @@ import java.util.HashSet; import java.util.Set; +import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.hadoop.conf.Configuration; @@ -29,6 +30,7 @@ import datawave.ingest.mapreduce.job.BulkIngestKey; import datawave.ingest.mapreduce.job.writer.ContextWriter; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; /** * This class differs from the parent in that when it sees a field name of ERROR it creates a category name using the field name and value. @@ -48,13 +50,10 @@ public class AtomErrorDataTypeHandler extends AtomDataTyp public static final String ERROR_FIELD = "ERROR"; public static final String STACK_TRACE_FIELD = "STACKTRACE"; public static final String EVENT_CONTENT_FIELD = "EVENT"; - public static final String COLUMN_VISIBILITY = "columnVisibility"; - private MarkingsHelper mHelper = null; - private ColumnVisibility defaultVisibility = null; + private AccessExpression defaultVisibility = null; private Configuration conf = null; private ErrorShardedIngestHelper errorHelper = null; - private MarkingFunctions markingFunctions; @Override public void setup(TaskAttemptContext context) { @@ -77,11 +76,7 @@ public long process(KEYIN key, RawRecordContainer record, Multimap markings = value.getMarkings(); + // value marking is 1st choice + if (markings != null && !markings.isEmpty()) { + return markings.toAccessExpression().getExpression(); + } ColumnVisibility visibility = event.getVisibility(); - if (value.getMarkings() != null && !value.getMarkings().isEmpty()) { - try { - visibility = markingFunctions.translateToColumnVisibility(value.getMarkings()); - } catch (MarkingFunctions.Exception e) { - throw new RuntimeException("Cannot convert record-level markings into a column visibility", e); - - } + if (visibility != null) { + // event marking is 2nd choice + return new String(visibility.getExpression()); } - return visibility == null ? defaultVisibility : visibility; + // default marking is the last resort + return defaultVisibility != null ? defaultVisibility.getExpression() : ""; } } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/dateindex/DateIndexDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/dateindex/DateIndexDataTypeHandler.java index 11a74c42307..bdd02f57535 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/dateindex/DateIndexDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/dateindex/DateIndexDataTypeHandler.java @@ -94,6 +94,7 @@ public class DateIndexDataTypeHandler implements DataTypeHandler, protected DateNormalizer dateNormalizer = new DateNormalizer(); protected ShardIdFactory shardIdFactory = null; protected TaskAttemptContext taskAttemptContext = null; + protected MarkingFunctions markingFunctions; public Set getDataTypes() { Set types = new HashSet<>(); @@ -148,6 +149,7 @@ public void setup(TaskAttemptContext context) { this.taskAttemptContext = context; this.conf = context.getConfiguration(); this.shardIdFactory = new ShardIdFactory(conf); + markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); String tableName = conf.get(DATEINDEX_TNAME, null); if (null == tableName) { @@ -364,7 +366,7 @@ private Value createDateIndexValue(int shard) { * @return the flattened visibility */ protected byte[] flatten(ColumnVisibility vis) { - return MarkingFunctions.Factory.createMarkingFunctions().flatten(vis); + return markingFunctions == null ? vis.flatten() : markingFunctions.flatten(vis); } public Text getDateIndexTableName() { diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDataTypeHandler.java index 7c70e7235a4..4638824ae8b 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDataTypeHandler.java @@ -1,5 +1,7 @@ package datawave.ingest.mapreduce.handler.edge; +import static java.nio.charset.StandardCharsets.UTF_8; + import static datawave.ingest.mapreduce.handler.shard.ShardedDataTypeHandler.METADATA_TABLE_LOADER_PRIORITY; import static datawave.ingest.mapreduce.handler.shard.ShardedDataTypeHandler.METADATA_TABLE_NAME; @@ -20,6 +22,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.ColumnVisibility; @@ -73,7 +76,9 @@ import datawave.ingest.metadata.RawRecordMetadata; import datawave.ingest.table.config.LoadDateTableConfigHelper; import datawave.ingest.time.Now; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.metadata.protobuf.EdgeMetadata; import datawave.metadata.protobuf.EdgeMetadata.MetadataValue; import datawave.metadata.protobuf.EdgeMetadata.MetadataValue.Metadata; @@ -141,7 +146,7 @@ public class ProtobufEdgeDataTypeHandler implements Exten protected String edgeTableName = null; protected String metadataTableName = null; protected boolean enableMetadata = false; - protected MarkingFunctions markingFunctions; + protected MarkingFunctions markingFunctions; protected Map edges = null; @@ -1327,15 +1332,13 @@ public static Map> getLocalityGroups(Configuration conf) { * the event container * @return the visibility as Text object */ - protected Text getVisibility(Map markings, RawRecordContainer event) { - try { - if (null == markings || markings.isEmpty()) { - return new Text(flatten(event.getVisibility())); - } else { - return new Text(flatten(markingFunctions.translateToColumnVisibility(markings))); - } - } catch (datawave.marking.MarkingFunctions.Exception e) { - throw new RuntimeException("Cannot convert markings into column visibility", e); + @SuppressWarnings("unchecked") + protected Text getVisibility(Markings markings, RawRecordContainer event) { + if (null == markings || markings.isEmpty()) { + return new Text(flatten(event.getVisibility())); + } else { + AccessExpression ae = AccessExpressionUtil.normalize(markings.toAccessExpression()); + return new Text(ae.getExpression().getBytes(UTF_8)); } } @@ -1363,7 +1366,7 @@ protected Text getDurationVisibility(EdgeDataBundle value) { * @return the flattened visibility */ protected byte[] flatten(ColumnVisibility vis) { - return markingFunctions.flatten(vis); + return markingFunctions == null ? vis.flatten() : markingFunctions.flatten(vis); } @Override diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/DurationValue.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/DurationValue.java index df3f9023476..33b9f093b48 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/DurationValue.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/DurationValue.java @@ -1,8 +1,7 @@ package datawave.ingest.mapreduce.handler.edge.define; -import java.util.Map; - import datawave.ingest.data.config.NormalizedContentInterface; +import datawave.marking.Markings; /** * Extracts the duration field specified by a Normalized Content Interface @@ -13,7 +12,7 @@ public class DurationValue { private int duration = -1; - private Map markings = null; + private Markings markings = null; public DurationValue(NormalizedContentInterface elapsedTimeNCI) { String durString = elapsedTimeNCI.getIndexedFieldValue(); @@ -36,7 +35,7 @@ public int getDuration() { } } - public Map getMarkings() { + public Markings getMarkings() { return this.markings; } } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/EdgeDataBundle.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/EdgeDataBundle.java index 77d25163774..28ba422499c 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/EdgeDataBundle.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/EdgeDataBundle.java @@ -3,7 +3,6 @@ import java.io.IOException; import java.util.Calendar; import java.util.List; -import java.util.Map; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.ColumnVisibility; @@ -20,7 +19,7 @@ import datawave.ingest.data.config.ingest.IngestHelperInterface; import datawave.ingest.mapreduce.handler.edge.define.VertexValue.ValueType; import datawave.marking.MarkingFunctions; -import datawave.marking.MarkingFunctions.Exception; +import datawave.marking.Markings; import datawave.util.time.DateHelper; /** @@ -34,43 +33,43 @@ public class EdgeDataBundle { // Input/Setup variables // final so you're not tempted to change them private final RawRecordContainer event; + private final MarkingFunctions markingFunctions; private VertexValue source; private VertexValue sink; - private EdgeDefinition edgeDefinition = null; + private EdgeDefinition edgeDefinition; private EdgeDirection edgeDirection = EdgeDirection.UNIDIRECTIONAL; - private String sourceMaskedValue = null; - private boolean hasMaskedSource = false; - private String sinkMaskedValue = null; - private boolean hasMaskedSink = false; + private String sourceMaskedValue; + private boolean hasMaskedSource; + private String sinkMaskedValue; + private boolean hasMaskedSink; // Variables for output - private boolean requiresMasking = false; - private boolean isDeleting = false; + private boolean requiresMasking; + private boolean isDeleting; private String edgeType; private String enrichedIndex; private String enrichedValue; private String edgeAttribute3; private String edgeAttribute2; - private long eventDate = 0; + private long eventDate; private IngestHelperInterface helper; - private Map markings = null; - private ColumnVisibility maskedVisibility = null; - private boolean forceMaskedVisibility = false; // if this is a masked event, but the + private Markings markings; + private ColumnVisibility maskedVisibility; + private boolean forceMaskedVisibility; // if this is a masked event, but the // the fields defined aren't masked, use the unmasked visibility // Duration value if defined - private DurationValue durationValue = null; + private DurationValue durationValue; private String loadDate; - private MarkingFunctions mf = null; private String uuid; private long activityDate; private boolean validActivityDate; public EdgeDataBundle(RawRecordContainer event, String typeName, String id, IngestHelperInterface helper) { - this.mf = MarkingFunctions.Factory.createMarkingFunctions(); + this.markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); this.event = event; this.eventDate = event.getDate(); this.edgeType = typeName; @@ -80,7 +79,7 @@ public EdgeDataBundle(RawRecordContainer event, String typeName, String id, Inge public EdgeDataBundle(EdgeDefinition edgeDef, NormalizedContentInterface ifaceSource, NormalizedContentInterface ifaceSink, RawRecordContainer event, IngestHelperInterface helper) { - this(event, edgeDef.getEdgeType().toString(), null, helper); + this(event, edgeDef.getEdgeType(), null, helper); this.setSource(new VertexValue(edgeDef.isUseRealm(), edgeDef.getSourceIndexedFieldRealm(), edgeDef.getSourceEventFieldRealm(), edgeDef.getSourceRelationship(), edgeDef.getSourceCollection(), ifaceSource)); @@ -95,7 +94,11 @@ public EdgeDataBundle(EdgeDefinition edgeDef, NormalizedContentInterface ifaceSo // even though event, etc. references are saved above, passing in the event // prevents future bug this.initFieldMasking(helper, event); - this.initMarkings(getSource().getMarkings(), getSink().getMarkings()); + try { + this.markings = markingFunctions.combine(getSource().getMarkings(), getSink().getMarkings()); + } catch (MarkingFunctions.Exception e) { + throw new RuntimeException("Could not combine markings", e); + } } private int getHour(long time) { @@ -140,23 +143,6 @@ public void setMaskedVisibility(ColumnVisibility maskedVisibility) { this.maskedVisibility = maskedVisibility; } - @SuppressWarnings("unchecked") - private void initMarkings(Map m1, Map m2) { - if (m1 != null) { - if (m2 != null) { - try { - this.markings = mf.combine(m1, m2); - } catch (Exception e) { - throw new RuntimeException("Unable to combine markings", e); - } - } else { - this.markings = m1; - } - } else if (m2 != null) { - this.markings = m2; - } - } - public int getDuration() { return null == this.getDurationValue() ? -1 : this.getDurationValue().getDuration(); } @@ -192,24 +178,19 @@ public Value getEdgeValue(boolean forwardEdge, EdgeKey.DATE_TYPE date_type) { } if (date_type == EdgeKey.DATE_TYPE.EVENT_ONLY) { - if (validActivityDate) { - builder.setBadActivityDate(false); - } else { - builder.setBadActivityDate(true); - } - + builder.setBadActivityDate(!validActivityDate); } // Set counts if (!this.isDeleting()) { - builder.setCount(1l); + builder.setCount(1L); } else { - builder.setCount(-1l); + builder.setCount(-1L); } // Set Hour Bitmask if (hour != -1) { builder.setHour(hour); } - if (forwardEdge == true) { + if (forwardEdge) { builder.setSourceValue(source.getValue(ValueType.EVENT)); builder.setSinkValue(sink.getValue(ValueType.EVENT)); } else { @@ -236,16 +217,11 @@ public Value getStatsActivityValue(boolean forwardEdge, EdgeKey.DATE_TYPE date_t } if (date_type == EdgeKey.DATE_TYPE.EVENT_ONLY) { - if (validActivityDate) { - builder.setBadActivityDate(false); - } else { - builder.setBadActivityDate(true); - } - + builder.setBadActivityDate(!validActivityDate); } List hours = EdgeValueHelper.getLongListForHour(hour, this.isDeleting()); builder.setHours(hours); - if (forwardEdge == true) { + if (forwardEdge) { builder.setSourceValue(source.getValue(ValueType.EVENT)); } else { builder.setSourceValue(sink.getValue(ValueType.EVENT)); @@ -276,7 +252,7 @@ public Value getDurationAsValue(boolean forwardEdge) { EdgeValueBuilder builder = datawave.edge.util.EdgeValue.newBuilder(); List duration = EdgeValueHelper.getLongListForDuration(this.getDuration(), this.isDeleting()); builder.setDuration(duration); - if (forwardEdge == true) { + if (forwardEdge) { builder.setSourceValue(source.getValue(ValueType.EVENT)); } else { builder.setSourceValue(sink.getValue(ValueType.EVENT)); @@ -385,11 +361,11 @@ public void setSink(VertexValue sink) { this.sink = sink; } - public Map getMarkings() { + public Markings getMarkings() { return this.markings; } - public void setMarkings(Map markings) { + public void setMarkings(Markings markings) { this.markings = markings; } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/VertexValue.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/VertexValue.java index bfbf4614323..6d8c1df3010 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/VertexValue.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/VertexValue.java @@ -1,8 +1,7 @@ package datawave.ingest.mapreduce.handler.edge.define; -import java.util.Map; - import datawave.ingest.data.config.NormalizedContentInterface; +import datawave.marking.Markings; /** * Combines a VertexDefinition with value obtained from a {@link datawave.ingest.data.config.NormalizedContentInterface}. @@ -23,7 +22,7 @@ public enum ValueType { private String indexedRealmLabel = null; private String eventRealmLabel = null; private String sourceIndex = null; - private Map markings = null; + private Markings markings = null; private String maskedValue = null; private boolean hasMaskedValue = false; private String relationshipType = null; @@ -50,7 +49,7 @@ public VertexValue(boolean useRealm, String indexedRealmLabel, String eventRealm } } - public Map getMarkings() { + public Markings getMarkings() { return this.markings; } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorDataTypeHandler.java index 0abad1bf986..f50d5592a50 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorDataTypeHandler.java @@ -1,10 +1,12 @@ package datawave.ingest.mapreduce.handler.error; +import static java.nio.charset.StandardCharsets.UTF_8; + import java.io.IOException; import java.io.PrintStream; import java.util.Date; -import java.util.Map; +import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.ColumnVisibility; @@ -109,7 +111,7 @@ public class ErrorDataTypeHandler implements ExtendedData private byte[] defaultVisibility = null; protected MarkingsHelper markingsHelper; - protected MarkingFunctions markingFunctions; + protected MarkingFunctions markingFunctions; private String tableName = null; @@ -133,11 +135,11 @@ public void setup(TaskAttemptContext context) { this.conf = context.getConfiguration(); - try { - Map defaultMarkings = markingsHelper.getDefaultMarkings(); - defaultVisibility = flatten(markingFunctions.translateToColumnVisibility(defaultMarkings)); - } catch (MarkingFunctions.Exception e) { - throw new IllegalArgumentException("Failed to convert default markings to a ColumnVisibility.", e); + if (markingsHelper != null && markingsHelper.getDefaultMarkings() != null) { + AccessExpression ae = markingsHelper.getDefaultMarkings().toAccessExpression(); + if (ae != null) { + defaultVisibility = ae.getExpression().getBytes(UTF_8); + } } // Initialize a UID builder based on the configuration @@ -320,12 +322,7 @@ private Key createKey(String row, Text colf, Text colq, byte[] vis, long ts) { protected byte[] getVisibility(RawRecordContainer event, NormalizedContentInterface value) { byte[] visibility; if (value != null && value.getMarkings() != null && !value.getMarkings().isEmpty()) { - try { - visibility = flatten(markingFunctions.translateToColumnVisibility(value.getMarkings())); - } catch (MarkingFunctions.Exception e) { - log.error("Failed to create visibility from markings, using default", e); - visibility = defaultVisibility; - } + visibility = flatten(value.getMarkings().toColumnVisibility()); } else if (event.getVisibility() != null) { visibility = flatten(event.getVisibility()); } else { @@ -335,14 +332,14 @@ protected byte[] getVisibility(RawRecordContainer event, NormalizedContentInterf } /** - * Create a flattened visibility, using the cache if possible + * Normalize a ColumnVisibility expression via AccessExpression and return the expression bytes. * * @param vis * the visibility - * @return the flattened visibility + * @return the normalized visibility bytes */ protected byte[] flatten(ColumnVisibility vis) { - return markingFunctions.flatten(vis); + return markingFunctions == null ? vis.flatten() : markingFunctions.flatten(vis); } @Override diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java index 6f416d02fff..9b613c85c3d 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java @@ -1,5 +1,7 @@ package datawave.ingest.mapreduce.handler.error; +import static java.nio.charset.StandardCharsets.UTF_8; + import java.io.IOException; import java.io.PrintStream; import java.util.ArrayList; @@ -7,6 +9,7 @@ import java.util.Date; import java.util.List; +import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; import org.apache.hadoop.conf.Configurable; @@ -41,7 +44,6 @@ import datawave.ingest.mapreduce.handler.shard.ShardedDataTypeHandler; import datawave.ingest.mapreduce.job.BulkIngestKey; import datawave.ingest.mapreduce.job.writer.ContextWriter; -import datawave.marking.MarkingFunctions; /** * Handler that take events with processing errors or fatal errors and dumps them into a processing error table. This table will be used for subsequent @@ -155,10 +157,11 @@ public void setup(TaskAttemptContext context) { tableName = conf.get(ERROR_PROP_PREFIX + SHARD_DINDX_NAME); setShardDictionaryIndexTableName(tableName == null ? null : new Text(tableName)); - try { - defaultVisibility = flatten(markingFunctions.translateToColumnVisibility(markingsHelper.getDefaultMarkings())); - } catch (Exception e) { - throw new IllegalArgumentException("Failed to parse security marking configuration", e); + if (markingsHelper != null && markingsHelper.getDefaultMarkings() != null) { + AccessExpression ae = markingsHelper.getDefaultMarkings().toAccessExpression(); + if (ae != null) { + defaultVisibility = ae.getExpression().getBytes(UTF_8); + } } log.info("ShardedErrorDataTypeHandler configured."); @@ -346,12 +349,7 @@ public static void getStackTrace(DataOutputBuffer buffer, Throwable e) { public byte[] getVisibility(RawRecordContainer event, NormalizedContentInterface value) { byte[] visibility; if (value != null && value.getMarkings() != null && !value.getMarkings().isEmpty()) { - try { - visibility = flatten(markingFunctions.translateToColumnVisibility(value.getMarkings())); - } catch (MarkingFunctions.Exception e) { - log.error("Failed to create visibility from markings, using default", e); - visibility = defaultVisibility; - } + visibility = flatten(value.getMarkings().toColumnVisibility()); } else if (event.getVisibility() != null) { visibility = flatten(event.getVisibility()); } else { diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/facet/FacetHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/facet/FacetHandler.java index 5e508dea644..d72497337ee 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/facet/FacetHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/facet/FacetHandler.java @@ -91,7 +91,7 @@ public class FacetHandler implements ExtendedDataTypeHand /* Instance variables */ - protected MarkingFunctions markingFunctions; + protected MarkingFunctions markingFunctions; protected ShardIdFactory shardIdFactory; protected TaskAttemptContext taskAttemptContext; diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java index 17a59dc6bd5..94ff1a0c06c 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java @@ -50,6 +50,7 @@ import datawave.ingest.util.BloomFilterWrapper; import datawave.ingest.util.DiskSpaceStarvationStrategy; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.query.model.Direction; import datawave.util.CompositeTimestamp; import datawave.util.TableName; @@ -219,7 +220,7 @@ public abstract class ShardedDataTypeHandler extends StatsDEnabledDataTyp private RawRecordMetadata metadata = null; private ShardIdFactory shardIdFactory = null; private LoadingCache dCache = null; - protected MarkingFunctions markingFunctions; + protected MarkingFunctions markingFunctions; protected IngestConfiguration ingestConfig = IngestConfigurationFactory.getIngestConfiguration(); // default setting is a standard index with uid and event keys @@ -1129,22 +1130,19 @@ public int getOffsetForYearIndex(String shard) { */ public byte[] getVisibility(RawRecordContainer event, NormalizedContentInterface value) { ColumnVisibility visibility = event.getVisibility(); - if (value.getMarkings() != null && !value.getMarkings().isEmpty()) { - try { - visibility = markingFunctions.translateToColumnVisibility(value.getMarkings()); - } catch (MarkingFunctions.Exception e) { - throw new RuntimeException("Cannot convert record-level markings into a column visibility", e); - } + Markings markings = value.getMarkings(); + if (markings != null && !markings.isEmpty()) { + visibility = markings.toColumnVisibility(); } return flatten(visibility); } /** - * Create a flattened visibility, using the cache if possible + * Normalize a ColumnVisibility expression via AccessExpression and return the expression bytes. * * @param vis * the visibility - * @return the flattened visibility + * @return the normalized visibility bytes */ protected byte[] flatten(ColumnVisibility vis) { return markingFunctions == null ? vis.flatten() : markingFunctions.flatten(vis); diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/tokenize/ExtendedContentIngestHelper.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/tokenize/ExtendedContentIngestHelper.java index d44e74af1f9..b3c0dd5388d 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/tokenize/ExtendedContentIngestHelper.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/tokenize/ExtendedContentIngestHelper.java @@ -5,7 +5,6 @@ import java.util.Map; import java.util.Set; -import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.log4j.Logger; @@ -170,89 +169,6 @@ public void addMetadataFromId(RawRecordContainer event, String id) { } } - public void inheritFromRootPayload(RawRecordContainer event, Map> metadata) { - // TODO: No-op here, but need to refactor the following in upstream - // if (metadata == null) - // return; - // - // IBaseDataObject payload = (IBaseDataObject) event.getAuxData(); - // if (payload != null) { - // for (String field : helper.getInheritedPayloadFields()) { - // if (metadata.containsKey(field) && (!payload.hasParameter(field))) { - // payload.putParameter(field, metadata.get(field)); - // } - // } - // } - } - - public void addMetadataFromParms(RawRecordContainer event, Map> metadata, String id) { - - for (Map.Entry> entry : metadata.entrySet()) { - CharSequence key = entry.getKey(); - for (Object value : entry.getValue()) { - if (value == null) - continue; - - if (helper.getUuids().contains(key)) { - event.getAltIds().add(String.valueOf(value)); - } - if (this.helper.getSecurityMarkingFieldDomainMap().containsKey(key)) { - addSecurityMetadataFromParms(key, value, event); - } - } - } - - Multimap newMetadata = HashMultimap.create(); - Multimap fieldParsers = helper.getMetadataFieldParsers(); - for (String field : fieldParsers.keySet()) { - if (metadata.containsKey(field) && metadata.get(field) != null) { - for (MetadataIdParser parser : fieldParsers.get(field)) { - for (Object v : metadata.get(field)) { - if (v == null) - continue; - try { - parser.addMetadata(event, newMetadata, v.toString()); - } catch (Exception e) { - log.error("Unable to apply " + parser + " to " + field, e); - event.addError(RawDataErrorNames.FIELD_EXTRACTION_ERROR); - } - } - } - } - } - - // If the ID is a UUID (i.e., contains no date), try to get - // event metadata from the parameters - if ((id != null) && (id.length() >= UUID_LENGTH) && id.matches(UUID_PATTERN)) { - fieldParsers = helper.getMetadataFieldUuidParsers(); - for (String field : fieldParsers.keySet()) { - if (metadata.containsKey(field) && metadata.get(field) != null) { - for (MetadataIdParser parser : fieldParsers.get(field)) { - for (Object v : metadata.get(field)) { - if (v == null) - continue; - try { - parser.addMetadata(event, newMetadata, v.toString()); - } catch (Exception e) { - log.error("Unable to apply " + parser + " to " + field, e); - event.addError(RawDataErrorNames.FIELD_EXTRACTION_ERROR); - } - } - } - } - } - } - } - - protected void addSecurityMetadataFromParms(CharSequence key, Object value, RawRecordContainer event) { - // If fieldName is a security marking field (as configured by EVENT_SECURITY_MARKING_FIELD_NAMES), - // then put the marking value into this.securityMarkings, where 'key' maps to the domain for the marking - // (as configured by EVENT_SECURITY_MARKING_FIELD_DOMAINS) - if (!StringUtils.isEmpty(key.toString()) && !StringUtils.isEmpty(value.toString())) { - event.addSecurityMarking(this.helper.getSecurityMarkingFieldDomainMap().get(key.toString()), value.toString()); - } - } - /** * Override the normalize call to enable event field value normalization * diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/data/RawRecordContainerImplTest.java b/warehouse/ingest-core/src/test/java/datawave/ingest/data/RawRecordContainerImplTest.java index fb92ff2492a..7201732a8a8 100644 --- a/warehouse/ingest-core/src/test/java/datawave/ingest/data/RawRecordContainerImplTest.java +++ b/warehouse/ingest-core/src/test/java/datawave/ingest/data/RawRecordContainerImplTest.java @@ -398,7 +398,7 @@ public void validate() { if (getVisibility() == null && getSecurityMarkings() == null) { if (markingsHelper != null) { setSecurityMarkings(markingsHelper.getDefaultMarkings()); - } else {} + } } if (getSecurityMarkings() == null) { diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/data/config/MarkingsHelperTest.java b/warehouse/ingest-core/src/test/java/datawave/ingest/data/config/MarkingsHelperTest.java index f4e90697c08..788863a0ecd 100644 --- a/warehouse/ingest-core/src/test/java/datawave/ingest/data/config/MarkingsHelperTest.java +++ b/warehouse/ingest-core/src/test/java/datawave/ingest/data/config/MarkingsHelperTest.java @@ -1,13 +1,12 @@ package datawave.ingest.data.config; -import java.util.Map; - import org.apache.hadoop.conf.Configuration; import org.junit.Assert; import org.junit.Test; import datawave.ingest.data.Type; -import datawave.marking.MarkingFunctions; +import datawave.marking.AccessExpressionMarkings; +import datawave.marking.Markings; public class MarkingsHelperTest { public static final String FIELD_NAME = "FIELDNAME1"; @@ -36,10 +35,10 @@ public void honorSpecifiedDatatypeFieldMarkings() { // use the same type name to retrieve a markings helper and retrieve the field marking MarkingsHelper markingsHelper = new MarkingsHelper.NoOp(conf, createType(typeName)); - Map fieldMarkingMap = markingsHelper.getFieldMarking(FIELD_NAME); + Markings fieldMarkings = markingsHelper.getFieldMarking(FIELD_NAME); - Assert.assertNotNull(fieldMarkingMap); - Assert.assertEquals(FIELD_MARKING_VALUE, fieldMarkingMap.get(MarkingFunctions.Default.COLUMN_VISIBILITY)); + Assert.assertNotNull(fieldMarkings); + Assert.assertEquals(FIELD_MARKING_VALUE, ((AccessExpressionMarkings) fieldMarkings).getAccessExpression().getExpression()); } private Configuration createConfWithFieldMarking(String typeName) { diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/data/config/NormalizedFieldAndValueTest.java b/warehouse/ingest-core/src/test/java/datawave/ingest/data/config/NormalizedFieldAndValueTest.java index f51e3eec718..e821d180e5c 100644 --- a/warehouse/ingest-core/src/test/java/datawave/ingest/data/config/NormalizedFieldAndValueTest.java +++ b/warehouse/ingest-core/src/test/java/datawave/ingest/data/config/NormalizedFieldAndValueTest.java @@ -1,11 +1,10 @@ package datawave.ingest.data.config; -import java.util.Map; - import org.junit.Assert; import org.junit.Test; import datawave.data.type.Type; +import datawave.marking.Markings; public class NormalizedFieldAndValueTest { @@ -19,7 +18,7 @@ public static class NonGroupedInstance implements NormalizedContentInterface { private String _eventFieldName; private String _eventFieldValue; - private Map _markings; + private Markings _markings; private Throwable _error; protected NonGroupedInstance() { @@ -84,7 +83,7 @@ public void setIndexedFieldValue(String val) { } @Override - public void setMarkings(Map markings) { + public void setMarkings(Markings markings) { _markings = markings; } @@ -95,7 +94,7 @@ public void setError(Throwable e) { } @Override - public Map getMarkings() { + public Markings getMarkings() { return _markings; } @@ -135,7 +134,7 @@ public static class GroupedInstance implements GroupedNormalizedContentInterface private String _eventFieldName; private String _eventFieldValue; - private Map _markings; + private Markings _markings; private Throwable _error; private boolean _grouped; @@ -208,7 +207,7 @@ public void setIndexedFieldValue(String val) { } @Override - public void setMarkings(Map markings) { + public void setMarkings(Markings markings) { _markings = markings; } @@ -219,7 +218,7 @@ public void setError(Throwable e) { } @Override - public Map getMarkings() { + public Markings getMarkings() { return _markings; } diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/data/config/ingest/FakeIngestHelper.java b/warehouse/ingest-core/src/test/java/datawave/ingest/data/config/ingest/FakeIngestHelper.java index 7fe022005c0..35858658c1b 100644 --- a/warehouse/ingest-core/src/test/java/datawave/ingest/data/config/ingest/FakeIngestHelper.java +++ b/warehouse/ingest-core/src/test/java/datawave/ingest/data/config/ingest/FakeIngestHelper.java @@ -1,7 +1,6 @@ package datawave.ingest.data.config.ingest; -import java.util.HashMap; -import java.util.Map; +import org.apache.accumulo.access.Access; import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; @@ -9,6 +8,7 @@ import datawave.ingest.data.RawRecordContainer; import datawave.ingest.data.config.NormalizedContentInterface; import datawave.ingest.data.config.NormalizedFieldAndValue; +import datawave.marking.AccessExpressionMarkings; /** * @@ -19,8 +19,7 @@ public class FakeIngestHelper extends BaseIngestHelper { @Override public Multimap getEventFields(RawRecordContainer event) { Multimap eventFields = HashMultimap.create(); - Map markings = new HashMap<>(); - markings.put("P", "PERSONAL"); + AccessExpressionMarkings markings = AccessExpressionMarkings.builder().accessExpression(Access.builder().build().newExpression("PERSONAL")).build(); eventFields.put("FAKE_FIELD", new NormalizedFieldAndValue("FAKE_FIELD", "fake value", markings)); return eventFields; diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/SimpleRawRecord.java b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/SimpleRawRecord.java index 9d8180b4021..28ffd3d8078 100644 --- a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/SimpleRawRecord.java +++ b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/SimpleRawRecord.java @@ -9,8 +9,8 @@ import java.util.Date; import java.util.HashMap; import java.util.Map; -import java.util.TreeMap; +import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.hadoop.io.Writable; @@ -20,6 +20,8 @@ import datawave.ingest.data.RawRecordContainer; import datawave.ingest.data.Type; import datawave.ingest.data.TypeRegistry; +import datawave.marking.AccessExpressionMarkings; +import datawave.marking.Markings; import datawave.util.CompositeTimestamp; /** @@ -29,7 +31,7 @@ public class SimpleRawRecord implements RawRecordContainer, Writable { private UIDBuilder uidBuilder = HashUID.builder(); - private Map securityMarkings = new TreeMap<>(); + private Markings securityMarkings = null; private UID id = uidBuilder.newId(); private Type dataType; private long timestamp = CompositeTimestamp.INVALID_TIMESTAMP; @@ -45,25 +47,15 @@ public class SimpleRawRecord implements RawRecordContainer, Writable { private boolean fatalError = false; @Override - public Map getSecurityMarkings() { + public Markings getSecurityMarkings() { return securityMarkings; } @Override - public void setSecurityMarkings(Map securityMarkings) { + public void setSecurityMarkings(Markings securityMarkings) { this.securityMarkings = securityMarkings; } - @Override - public void addSecurityMarking(String domain, String marking) { - securityMarkings.put(domain, marking); - } - - @Override - public boolean hasSecurityMarking(String domain, String marking) { - return marking.equals(securityMarkings.get(domain)); - } - @Override public UID getId() { return id; @@ -244,7 +236,14 @@ public long getDataOutputSize() { @Override public void write(DataOutput dataOutput) throws IOException { - TestWritableUtil.writeMap(securityMarkings, dataOutput); + String expr = ""; + if (securityMarkings != null) { + AccessExpression ae = securityMarkings.toAccessExpression(); + if (ae != null) { + expr = ae.getExpression(); + } + } + dataOutput.writeUTF(expr); id.write(dataOutput); dataOutput.writeUTF(dataType.typeName()); @@ -269,7 +268,12 @@ public void write(DataOutput dataOutput) throws IOException { @Override public void readFields(DataInput dataInput) throws IOException { - securityMarkings = TestWritableUtil.readMap(dataInput); + String expr = dataInput.readUTF(); + if (!expr.isEmpty()) { + securityMarkings = AccessExpressionMarkings.builder().columnVisibility(expr).build(); + } else { + securityMarkings = null; + } id = uidBuilder.newId(); id.readFields(dataInput); diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDeleteModeTest.java b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDeleteModeTest.java index 5e96b93168e..ecceef27fb5 100644 --- a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDeleteModeTest.java +++ b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDeleteModeTest.java @@ -131,7 +131,7 @@ public void setup() { private RawRecordContainer getEvent(Configuration conf) { RawRecordContainerImpl myEvent = new RawRecordContainerImpl(); - myEvent.addSecurityMarking("columnVisibility", "PRIVATE"); + myEvent.setVisibility("PRIVATE"); myEvent.setDataType(type); myEvent.setId(UID.builder().newId()); myEvent.setConf(conf); diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDeletePreconditionTest.java b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDeletePreconditionTest.java index fbbc463eb24..7b8c2868ad4 100644 --- a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDeletePreconditionTest.java +++ b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDeletePreconditionTest.java @@ -62,7 +62,7 @@ public void setup() { private RawRecordContainer getEvent(Configuration conf) { RawRecordContainerImpl myEvent = new RawRecordContainerImpl(); - myEvent.addSecurityMarking("columnVisibility", "PRIVATE"); + myEvent.setVisibility("PRIVATE"); myEvent.setDataType(type); myEvent.setId(UID.builder().newId()); myEvent.setConf(conf); diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDirectionTest.java b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDirectionTest.java index 90fbb1a720a..aa2f79c44c4 100644 --- a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDirectionTest.java +++ b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDirectionTest.java @@ -319,7 +319,7 @@ public void testUniDirectionWithJexlCondition() { private RawRecordContainer getEvent(Type type) { RawRecordContainerImpl myEvent = new RawRecordContainerImpl(); - myEvent.addSecurityMarking("columnVisibility", "PRIVATE"); + myEvent.setVisibility("PRIVATE"); myEvent.setDataType(type); myEvent.setId(UID.builder().newId()); myEvent.setAltIds(Collections.singleton("0016dd72-0000-827d-dd4d-001b2163ba09")); diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgePreconditionTest.java b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgePreconditionTest.java index 8ff47a246f9..c1a56a8c13a 100644 --- a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgePreconditionTest.java +++ b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgePreconditionTest.java @@ -72,7 +72,7 @@ public void setup() { private RawRecordContainer getEvent(Configuration conf) { RawRecordContainerImpl myEvent = new RawRecordContainerImpl(); - myEvent.addSecurityMarking("columnVisibility", "PRIVATE"); + myEvent.setVisibility("PRIVATE"); myEvent.setDataType(type); myEvent.setId(UID.builder().newId()); myEvent.setAltIds(Collections.singleton("0016dd72-0000-827d-dd4d-001b2163ba09")); diff --git a/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java b/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java index f09c886b623..388f9f2b78d 100644 --- a/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java +++ b/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java @@ -1,10 +1,10 @@ package datawave.ingest.csv.mr.input; import java.io.IOException; -import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; +import org.apache.accumulo.access.AccessExpression; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Text; @@ -27,8 +27,10 @@ import datawave.ingest.input.reader.event.EventFixer; import datawave.ingest.metadata.id.MetadataIdParser; import datawave.ingest.validation.EventValidator; +import datawave.marking.AccessExpressionMarkings; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; -import datawave.marking.MarkingFunctionsFactory; +import datawave.marking.Markings; public class CSVRecordReader extends CSVReaderBase implements EventFixer { @@ -42,7 +44,7 @@ public class CSVRecordReader extends CSVReaderBase implements EventFixer { private ExtendedCSVHelper csvHelper; private DataTypeOverrideHelper dataTypeHelper; - private Map securityMarkings; + private Markings securityMarkings; @Override public void initialize(InputSplit genericSplit, TaskAttemptContext context) throws IOException { @@ -126,12 +128,7 @@ public RawRecordContainer getEvent() { protected void decorateEvent() { if (null != this.securityMarkings && !this.securityMarkings.isEmpty()) { event.setSecurityMarkings(securityMarkings); - try { - event.setVisibility(MarkingFunctionsFactory.createMarkingFunctions().translateToColumnVisibility(securityMarkings)); - } catch (MarkingFunctions.Exception e) { - log.error("Could not set default ColumnVisibility for the event", e); - throw new RuntimeException(e); - } + event.setVisibility(securityMarkings.toColumnVisibility()); } // now validate @@ -174,14 +171,16 @@ protected void checkField(String fieldName, String fieldValue) { } // If fieldName is a security marking field (as configured by EVENT_SECURITY_MARKING_FIELD_NAMES), - // then put the marking value into this.securityMarkings, where the key is the field name for the marking - // (as configured by EVENT_SECURITY_MARKING_FIELD_DOMAINS) + // then accumulate the marking value into this.securityMarkings if (this.csvHelper.getSecurityMarkingFieldDomainMap().containsKey(fieldName)) { - if (null == this.securityMarkings) { - this.securityMarkings = new HashMap<>(); - } if (!StringUtils.isEmpty(fieldValue)) { - this.securityMarkings.put(this.csvHelper.getSecurityMarkingFieldDomainMap().get(fieldName), fieldValue); + AccessExpression ae = AccessExpressionUtil.normalize(fieldValue); + AccessExpressionMarkings newMarking = AccessExpressionMarkings.builder().accessExpression(ae).build(); + if (securityMarkings == null) { + securityMarkings = newMarking; + } else { + securityMarkings = new MarkingFunctions.Default().combine(securityMarkings, newMarking); + } } } // Now lets add metadata extracted from the parsers diff --git a/warehouse/ingest-csv/src/test/java/datawave/ingest/csv/NormalizedContentInterfaceTest.java b/warehouse/ingest-csv/src/test/java/datawave/ingest/csv/NormalizedContentInterfaceTest.java index cee757075ed..035e4dcb295 100644 --- a/warehouse/ingest-csv/src/test/java/datawave/ingest/csv/NormalizedContentInterfaceTest.java +++ b/warehouse/ingest-csv/src/test/java/datawave/ingest/csv/NormalizedContentInterfaceTest.java @@ -1,8 +1,6 @@ package datawave.ingest.csv; import java.io.IOException; -import java.util.HashMap; -import java.util.Map; import java.util.Objects; import org.apache.hadoop.conf.Configuration; @@ -29,6 +27,8 @@ import datawave.ingest.data.config.NormalizedFieldAndValue; import datawave.ingest.data.config.ingest.BaseIngestHelper; import datawave.ingest.data.config.ingest.IngestHelperInterface; +import datawave.marking.ColumnVisibilitySecurityMarking; +import datawave.marking.Markings; public class NormalizedContentInterfaceTest { private static final Logger log = LoggerFactory.getLogger(NormalizedContentInterfaceTest.class); @@ -178,10 +178,10 @@ private NormalizedContentInterface createNormalizedContent(String field, String return fieldAndValue; } - private Map createDefaultMarkings() { - Map defaultMarkings = new HashMap<>(1); - defaultMarkings.put("columnVisibility", "PRIVATE"); - return defaultMarkings; + private Markings createDefaultMarkings() { + ColumnVisibilitySecurityMarking sm = new ColumnVisibilitySecurityMarking(); + sm.setColumnVisibility("PRIVATE"); + return sm.toMarkings(); } private void compare(Multimap actualValues, Multimap expectedValues) { diff --git a/warehouse/ingest-scripts/src/main/resources/bin/ingest/findJars.sh b/warehouse/ingest-scripts/src/main/resources/bin/ingest/findJars.sh index 7a1a376e530..a7b487fc275 100644 --- a/warehouse/ingest-scripts/src/main/resources/bin/ingest/findJars.sh +++ b/warehouse/ingest-scripts/src/main/resources/bin/ingest/findJars.sh @@ -84,6 +84,7 @@ LUCENE_JAR=$LUCENE_JAR:$(findJar lucene-queryparser) LUCENE_JAR=$LUCENE_JAR:$(findJar lucene-analyzers-common) THRIFT_JAR=$(findJar libthrift) AC_CORE_JAR=$(findAccumuloJar accumulo-core) +AC_ACCESS_JAR=$(findJar accumulo-access) AC_SERVER_JAR=$(findAccumuloJar accumulo-server-base) AC_START_JAR=$(findAccumuloJar accumulo-start) AC_MAPRED_JAR=$(findAccumuloJar accumulo-hadoop-mapreduce) diff --git a/warehouse/ingest-scripts/src/main/resources/bin/ingest/ingest-libs.sh b/warehouse/ingest-scripts/src/main/resources/bin/ingest/ingest-libs.sh index b4fdb395d39..8eabe24b185 100755 --- a/warehouse/ingest-scripts/src/main/resources/bin/ingest/ingest-libs.sh +++ b/warehouse/ingest-scripts/src/main/resources/bin/ingest/ingest-libs.sh @@ -57,6 +57,7 @@ CLASSPATH=${CLASSPATH}:${LOG4J2_SLF4J_JAR} CLASSPATH=${CLASSPATH}:${LUCENE_JAR} CLASSPATH=${CLASSPATH}:${THRIFT_JAR} CLASSPATH=${CLASSPATH}:${AC_CORE_JAR} +CLASSPATH=${CLASSPATH}:${AC_ACCESS_JAR} CLASSPATH=${CLASSPATH}:${AC_MAPRED_JAR} CLASSPATH=${CLASSPATH}:${AC_SERVER_JAR} CLASSPATH=${CLASSPATH}:${AC_START_JAR} diff --git a/warehouse/ingest-ssdeep/src/main/java/datawave/ingest/mapreduce/handler/ssdeep/SSDeepIndexHandler.java b/warehouse/ingest-ssdeep/src/main/java/datawave/ingest/mapreduce/handler/ssdeep/SSDeepIndexHandler.java index d08dfd5667b..eff5f2c3ab0 100644 --- a/warehouse/ingest-ssdeep/src/main/java/datawave/ingest/mapreduce/handler/ssdeep/SSDeepIndexHandler.java +++ b/warehouse/ingest-ssdeep/src/main/java/datawave/ingest/mapreduce/handler/ssdeep/SSDeepIndexHandler.java @@ -8,7 +8,6 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Text; @@ -70,7 +69,7 @@ public class SSDeepIndexHandler implements ExtendedDataTy protected Text ssdeepIndexTableName; - protected MarkingFunctions markingFunctions; + protected MarkingFunctions markingFunctions; protected TaskAttemptContext taskAttemptContext; @@ -135,10 +134,6 @@ public RawRecordMetadata getMetadata() { return null; } - protected byte[] flatten(ColumnVisibility vis) { - return markingFunctions == null ? vis.flatten() : markingFunctions.flatten(vis); - } - @Override public long process(KEYIN key, RawRecordContainer event, Multimap fields, TaskInputOutputContext context, ContextWriter contextWriter) diff --git a/warehouse/keyword-common/src/main/java/datawave/util/keyword/DefaultTagCloudUtils.java b/warehouse/keyword-common/src/main/java/datawave/util/keyword/DefaultTagCloudUtils.java index 3ad1b52735f..1a039688f7a 100644 --- a/warehouse/keyword-common/src/main/java/datawave/util/keyword/DefaultTagCloudUtils.java +++ b/warehouse/keyword-common/src/main/java/datawave/util/keyword/DefaultTagCloudUtils.java @@ -1,8 +1,8 @@ package datawave.util.keyword; import java.io.Serializable; +import java.nio.charset.StandardCharsets; import java.util.Collection; -import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.Map; @@ -13,6 +13,8 @@ import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.commons.lang3.StringUtils; +import datawave.marking.MarkingFunctions; + /** * Default implementations for pluggable utilities for generating tag clouds, includes mechanisms to partition keywords into separate tag clouds, combine or * merge visibility strings, and calculate scores, source collections and frequencies of individual keywords based on observed results @@ -20,18 +22,23 @@ public class DefaultTagCloudUtils implements TagCloudUtils, Serializable { private static final long serialVersionUID = 652771994052429009L; private static final String MULTI_VALUE_SEPARATOR = ","; + private MarkingFunctions markingFunctions; @Override - public Map generateCombinedVisibility(Set visibilities) { - final StringBuilder b = new StringBuilder(); - visibilities.forEach(x -> b.append("(").append(x).append(")&")); - if (b.length() > 0) { - b.setLength(b.length() - 1); - ColumnVisibility cv = new ColumnVisibility(b.toString()); - ColumnVisibility flat = new ColumnVisibility(cv.flatten()); - return Map.of("visibility", flat.toString()); - } else { - return Collections.emptyMap(); + public String generateCombinedVisibility(Set visibilities) { + if (visibilities == null || visibilities.isEmpty()) { + return null; + } + try { + if (null == markingFunctions) { + markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); + } + // Calculate the columnVisibility for this key from the combiner. + ColumnVisibility columnVisibility = markingFunctions + .combineVisibilities(visibilities.stream().map(ColumnVisibility::new).collect(Collectors.toList())); + return new String(columnVisibility.getExpression(), StandardCharsets.UTF_8); + } catch (Exception e) { + throw new RuntimeException(e); } } diff --git a/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloud.java b/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloud.java index 584803325f9..aad64d480fa 100644 --- a/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloud.java +++ b/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloud.java @@ -1,5 +1,7 @@ package datawave.util.keyword; +import static datawave.marking.AccessExpressionMarkings.ACCESS; + import java.util.ArrayList; import java.util.Collection; import java.util.Comparator; @@ -12,6 +14,8 @@ import java.util.SortedSet; import java.util.TreeSet; +import org.apache.accumulo.access.AccessExpression; + import com.google.gson.Gson; /** A tag cloud - a collection of tags that have a keyword, score, frequency and list of sources from which they originated */ @@ -22,8 +26,8 @@ public class TagCloud { /** metadata for the cloud */ final Map metadata; - /** the 'visibility' of this cloud */ - final Map visibility; + /** the combined visibility expression string for this cloud (stored as String for Gson serialization) */ + final String visibility; /** the sorted set of keywords in this cloud, including scores and sources */ final SortedSet results; @@ -34,11 +38,11 @@ public class TagCloud { * @param metadata * metadata for the tag cloud * @param visibility - * the visibility of the cloud, some form of visibility marking + * the combined ColumnVisibility for the cloud, or {@code null} * @param results * the entries that belong in the tag cloud. */ - protected TagCloud(Map metadata, Map visibility, SortedSet results) { + protected TagCloud(Map metadata, String visibility, SortedSet results) { this.metadata = metadata; this.visibility = visibility; this.results = results; @@ -48,8 +52,12 @@ public Map getMetadata() { return metadata; } - public Map getVisibility() { - return visibility; + /** Return the combined visibility as an {@link AccessExpression}. */ + public AccessExpression getVisibility() { + if (visibility == null || visibility.isEmpty()) { + return null; + } + return ACCESS.newExpression(visibility); } public Collection getResults() { @@ -213,7 +221,7 @@ public List build() { final SortedSet results = new TreeSet<>(comparator); results.addAll(e.getValue()); String partition = getPartition(e.getKey()); - Map visibility = utils.generateCombinedVisibility(visibilities.get(partition)); + String visibility = utils.generateCombinedVisibility(visibilities.get(partition)); Map tagCloudMetadata = utils.generateCombinedMetadata(this.metadata.get(partition)); tagClouds.add(new TagCloud(tagCloudMetadata, visibility, results)); } diff --git a/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloudUtils.java b/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloudUtils.java index ecfb1f80d39..d40d316641d 100644 --- a/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloudUtils.java +++ b/warehouse/keyword-common/src/main/java/datawave/util/keyword/TagCloudUtils.java @@ -11,13 +11,13 @@ */ public interface TagCloudUtils { /** - * Aggregate visibilities by concatenating them together with an AND ('&'), and then flatten them using the Accumulo column visibility class. + * Aggregate visibilities by concatenating them together with an AND ('&'), and then normalizing the resulting expression. * * @param visibilities * a set of valid visibility strings. - * @return a single visibility + * @return a combined visibility string, or {@code null} if there are no visibilities */ - Map generateCombinedVisibility(Set visibilities); + String generateCombinedVisibility(Set visibilities); /** * Aggregate metadata from multiple sources, combining multi-valued entries into a flattened string diff --git a/warehouse/keyword-common/src/test/resources/MarkingFunctionsContext.xml b/warehouse/keyword-common/src/test/resources/MarkingFunctionsContext.xml new file mode 100644 index 00000000000..a57f6816cab --- /dev/null +++ b/warehouse/keyword-common/src/test/resources/MarkingFunctionsContext.xml @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + diff --git a/warehouse/query-core/src/main/java/datawave/core/iterators/FieldIndexCountingIteratorPerVisibility.java b/warehouse/query-core/src/main/java/datawave/core/iterators/FieldIndexCountingIteratorPerVisibility.java index 9fa11756fe3..4a532d353b0 100644 --- a/warehouse/query-core/src/main/java/datawave/core/iterators/FieldIndexCountingIteratorPerVisibility.java +++ b/warehouse/query-core/src/main/java/datawave/core/iterators/FieldIndexCountingIteratorPerVisibility.java @@ -98,6 +98,7 @@ public class FieldIndexCountingIteratorPerVisibility extends WrappingIterator im private Set columnVisibilities = Sets.newHashSet(); private TreeMap keyCache = null; + private MarkingFunctions markingFunctions; // ------------------------------------------------------------------------- // ------------- Constructors @@ -562,10 +563,13 @@ private Map buildReturnKeys() { } } - ColumnVisibility cv = null; + ColumnVisibility cv; try { + if (null == markingFunctions) { + markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); + } // Calculate the columnVisibility for this key from the combiner. - cv = MarkingFunctions.Factory.createMarkingFunctions().combine(columnVisibilities); + cv = markingFunctions.combineVisibilities(columnVisibilities); } catch (Exception e) { log.error("Could not create combined columnVisibility for the count", e); return null; diff --git a/warehouse/query-core/src/main/java/datawave/core/iterators/GlobalIndexDateSummaryIterator.java b/warehouse/query-core/src/main/java/datawave/core/iterators/GlobalIndexDateSummaryIterator.java index 6329697cc99..ec5e03d77ac 100644 --- a/warehouse/query-core/src/main/java/datawave/core/iterators/GlobalIndexDateSummaryIterator.java +++ b/warehouse/query-core/src/main/java/datawave/core/iterators/GlobalIndexDateSummaryIterator.java @@ -201,6 +201,7 @@ protected static class TermInfoSummary { private String date = null; private Map summary = new HashMap<>(); private Map> columnVisibilitiesMap = Maps.newHashMap(); + protected MarkingFunctions markingFunctions; public TermInfoSummary(String fieldValue, String fieldName, String date) { this.fieldValue = fieldValue; @@ -255,8 +256,11 @@ public Map getKeyValues() throws IOException { // Calculate the ColumnVisibility for this key from the combiner. Set columnVisibilities = this.columnVisibilitiesMap.get(datatype); + if (null == markingFunctions) { + markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); + } // Note that the access controls found in the combined ColumnVisibility will be pulled out appropriately here - ColumnVisibility cv = MarkingFunctions.Factory.createMarkingFunctions().combine(columnVisibilities); + ColumnVisibility cv = markingFunctions.combineVisibilities(columnVisibilities); // Create a new Key compatible with the shardIndex key format Key k = new Key(this.fieldValue, this.fieldName, this.date + '\0' + datatype, new String(cv.getExpression())); diff --git a/warehouse/query-core/src/main/java/datawave/core/iterators/ResultCountingIterator.java b/warehouse/query-core/src/main/java/datawave/core/iterators/ResultCountingIterator.java index 04027b8beac..59d87b3cb0b 100644 --- a/warehouse/query-core/src/main/java/datawave/core/iterators/ResultCountingIterator.java +++ b/warehouse/query-core/src/main/java/datawave/core/iterators/ResultCountingIterator.java @@ -66,6 +66,7 @@ public long read() { private String threadName = null; protected Set columnVisibilities = Sets.newHashSet(); + protected MarkingFunctions markingFunctions; public ResultCountingIterator() { threadName = Thread.currentThread().getName(); @@ -206,16 +207,18 @@ public Value getTopValue() { sw.start(); - ColumnVisibility cv = null; - + if (null == markingFunctions) { + markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); + } + ColumnVisibility columnVisibility; try { - cv = MarkingFunctions.Factory.createMarkingFunctions().combine(columnVisibilities); + columnVisibility = markingFunctions.combineVisibilities(columnVisibilities); } catch (MarkingFunctions.Exception e) { log.error("Could not create combined columnVisibility for the count", e); return null; } - ResultCountTuple result = new ResultCountTuple(this.count, cv); + ResultCountTuple result = new ResultCountTuple(this.count, columnVisibility); ByteArrayOutputStream baos = new ByteArrayOutputStream(); Output kryoOutput = new Output(baos); kryo.writeObject(kryoOutput, result); diff --git a/warehouse/query-core/src/main/java/datawave/query/attributes/Attribute.java b/warehouse/query-core/src/main/java/datawave/query/attributes/Attribute.java index 567d61e779a..8c4d33729f0 100644 --- a/warehouse/query-core/src/main/java/datawave/query/attributes/Attribute.java +++ b/warehouse/query-core/src/main/java/datawave/query/attributes/Attribute.java @@ -8,6 +8,7 @@ import java.util.Arrays; import java.util.Collection; +import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.data.ArrayByteSequence; import org.apache.accumulo.core.data.ByteSequence; import org.apache.accumulo.core.data.Key; @@ -26,6 +27,7 @@ import datawave.core.cache.CaffeineClassCache; import datawave.core.cache.ClassCache; +import datawave.marking.AccessExpressionUtil; import datawave.query.Constants; import datawave.query.jexl.DatawaveJexlContext; @@ -68,6 +70,15 @@ public ColumnVisibility getColumnVisibility() { return Constants.EMPTY_VISIBILITY; } + /** + * Get the access expression for this attribute, converted from the column visibility. + * + * @return the access expression + */ + public AccessExpression getAccessExpression() { + return AccessExpressionUtil.toAccessExpression(getColumnVisibility()); + } + /** * Get a copy byte array that backs the {@link ColumnVisibility}. This avoids the expensive parse call found in the default constructor for the * ColumnVisibility. diff --git a/warehouse/query-core/src/main/java/datawave/query/attributes/AttributeBag.java b/warehouse/query-core/src/main/java/datawave/query/attributes/AttributeBag.java index 130a91beb24..abde462a435 100644 --- a/warehouse/query-core/src/main/java/datawave/query/attributes/AttributeBag.java +++ b/warehouse/query-core/src/main/java/datawave/query/attributes/AttributeBag.java @@ -2,14 +2,13 @@ import java.io.Serializable; import java.util.Collection; +import java.util.stream.Collectors; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.commons.lang.mutable.MutableLong; import org.apache.log4j.Logger; -import com.google.common.collect.Sets; - import datawave.marking.MarkingFunctions; import datawave.marking.MarkingFunctions.Exception; @@ -19,11 +18,15 @@ public abstract class AttributeBag> extends Attribute private static final Logger log = Logger.getLogger(AttributeBag.class); protected long shardTimestamp = Long.MAX_VALUE; protected boolean validMetadata = false; + protected MarkingFunctions markingFunctions; private static final long ONE_DAY_MS = 1000l * 60 * 60 * 24; - public MarkingFunctions getMarkingFunctions() { - return MarkingFunctions.Factory.createMarkingFunctions(); + public MarkingFunctions getMarkingFunctions() { + if (null == markingFunctions) { + markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); + } + return markingFunctions; } protected AttributeBag() { @@ -42,8 +45,8 @@ public void invalidateMetadata() { this.validMetadata = false; } - public boolean isValidMetadata() { - return (this.validMetadata && isMetadataSet()); + public boolean isNotValidMetadata() { + return (!this.validMetadata || !isMetadataSet()); } public abstract Collection>> getAttributes(); @@ -53,14 +56,14 @@ public boolean isValidMetadata() { @Override public long getTimestamp() { // calling isMetadataSet first to update the metadata as needed - if (isValidMetadata() == false) + if (isNotValidMetadata()) this.updateMetadata(); return super.getTimestamp(); } @Override public ColumnVisibility getColumnVisibility() { - if (isValidMetadata() == false) + if (isNotValidMetadata()) this.updateMetadata(); return super.getColumnVisibility(); } @@ -78,11 +81,9 @@ private void updateMetadata() { } protected ColumnVisibility combineAndSetColumnVisibilities(Collection>> attributes) throws Exception { - Collection columnVisibilities = Sets.newHashSet(); - for (Attribute attr : attributes) { - columnVisibilities.add(attr.getColumnVisibility()); - } - return MarkingFunctions.Factory.createMarkingFunctions().combine(columnVisibilities); + Collection visibilities = attributes.stream().map(Attribute::getColumnVisibility).collect(Collectors.toList()); + markingFunctions = getMarkingFunctions(); + return markingFunctions.combineVisibilities(visibilities); } private long updateTimestamps() { diff --git a/warehouse/query-core/src/main/java/datawave/query/attributes/Attributes.java b/warehouse/query-core/src/main/java/datawave/query/attributes/Attributes.java index 7a83a35ad62..4bf62e1a1bf 100644 --- a/warehouse/query-core/src/main/java/datawave/query/attributes/Attributes.java +++ b/warehouse/query-core/src/main/java/datawave/query/attributes/Attributes.java @@ -20,7 +20,6 @@ import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; -import datawave.marking.MarkingFunctions; import datawave.query.collections.FunctionalSet; import datawave.query.jexl.DatawaveJexlContext; @@ -38,10 +37,6 @@ public class Attributes extends AttributeBag implements Serializable */ private boolean trackSizes; - public MarkingFunctions getMarkingFunctions() { - return MarkingFunctions.Factory.createMarkingFunctions(); - } - protected Attributes() { this(true); } diff --git a/warehouse/query-core/src/main/java/datawave/query/attributes/Document.java b/warehouse/query-core/src/main/java/datawave/query/attributes/Document.java index a3a76c51b0f..c17aec1f0ad 100644 --- a/warehouse/query-core/src/main/java/datawave/query/attributes/Document.java +++ b/warehouse/query-core/src/main/java/datawave/query/attributes/Document.java @@ -34,6 +34,7 @@ import com.google.common.collect.Sets; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.query.Constants; import datawave.query.collections.FunctionalSet; import datawave.query.composite.CompositeMetadata; @@ -80,16 +81,13 @@ public Long load(Text row) { private static final long ONE_DAY_MS = 1000L * 60 * 60 * 24; - public MarkingFunctions getMarkingFunctions() { - return MarkingFunctions.Factory.createMarkingFunctions(); - } - - public Map getMarkings() { + public Markings getMarkings() { try { - MarkingFunctions markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); + markingFunctions = getMarkingFunctions(); return markingFunctions.translateFromColumnVisibility(getColumnVisibility()); - } catch (MarkingFunctions.Exception e) {} - return Collections.emptyMap(); + } catch (MarkingFunctions.Exception ignored) { + return null; + } } public Document() { diff --git a/warehouse/query-core/src/main/java/datawave/query/common/grouping/AbstractAggregator.java b/warehouse/query-core/src/main/java/datawave/query/common/grouping/AbstractAggregator.java index 97f5a176a95..4fa5053d375 100644 --- a/warehouse/query-core/src/main/java/datawave/query/common/grouping/AbstractAggregator.java +++ b/warehouse/query-core/src/main/java/datawave/query/common/grouping/AbstractAggregator.java @@ -2,7 +2,7 @@ import java.util.Set; -import org.apache.accumulo.core.security.ColumnVisibility; +import org.apache.accumulo.access.AccessExpression; import datawave.query.attributes.Attribute; @@ -32,7 +32,7 @@ public String getField() { } @Override - public abstract Set getColumnVisibilities(); + public abstract Set getAccessExpressions(); @Override public abstract AGGREGATE getAggregation(); diff --git a/warehouse/query-core/src/main/java/datawave/query/common/grouping/Aggregator.java b/warehouse/query-core/src/main/java/datawave/query/common/grouping/Aggregator.java index 1b9329e6adc..70f8d60fc32 100644 --- a/warehouse/query-core/src/main/java/datawave/query/common/grouping/Aggregator.java +++ b/warehouse/query-core/src/main/java/datawave/query/common/grouping/Aggregator.java @@ -3,7 +3,7 @@ import java.util.Collection; import java.util.Set; -import org.apache.accumulo.core.security.ColumnVisibility; +import org.apache.accumulo.access.AccessExpression; import datawave.query.attributes.Attribute; @@ -30,11 +30,11 @@ public interface Aggregator { String getField(); /** - * Returns an unmodifiable set of all distinct column visibilities for each attribute aggregated into this aggregator. Possibly empty, but never null. + * Returns an unmodifiable set of all distinct access expressions for each attribute aggregated into this aggregator. Possibly empty, but never null. * - * @return a set of the column visibilities + * @return a set of the access expressions */ - Set getColumnVisibilities(); + Set getAccessExpressions(); /** * Return the aggregation result. diff --git a/warehouse/query-core/src/main/java/datawave/query/common/grouping/AverageAggregator.java b/warehouse/query-core/src/main/java/datawave/query/common/grouping/AverageAggregator.java index 976dfda7b96..161f47609ec 100644 --- a/warehouse/query-core/src/main/java/datawave/query/common/grouping/AverageAggregator.java +++ b/warehouse/query-core/src/main/java/datawave/query/common/grouping/AverageAggregator.java @@ -7,7 +7,7 @@ import java.util.HashSet; import java.util.Set; -import org.apache.accumulo.core.security.ColumnVisibility; +import org.apache.accumulo.access.AccessExpression; import org.apache.commons.lang.builder.ToStringBuilder; import datawave.query.attributes.Attribute; @@ -37,26 +37,26 @@ public class AverageAggregator extends AbstractAggregator { private BigDecimal average; /** - * The column visibilities of all attributes aggregated. + * The access expressions of all attributes aggregated. */ - private final Set columnVisibilities; + private final Set accessExpressions; public static AverageAggregator of(String field, TypeAttribute numerator, TypeAttribute divisor) { - return new AverageAggregator(field, numerator.getType().getDelegate(), divisor.getType().getDelegate(), numerator.getColumnVisibility()); + return new AverageAggregator(field, numerator.getType().getDelegate(), divisor.getType().getDelegate(), numerator.getAccessExpression()); } public AverageAggregator(String field) { super(field); - this.columnVisibilities = new HashSet<>(); + this.accessExpressions = new HashSet<>(); } - private AverageAggregator(String field, BigDecimal numerator, BigDecimal divisor, ColumnVisibility columnVisibility) { + private AverageAggregator(String field, BigDecimal numerator, BigDecimal divisor, AccessExpression expression) { this(field); this.numerator = numerator; this.divisor = divisor; this.average = numerator.divide(divisor, MATH_CONTEXT); - if (columnVisibility != null) { - this.columnVisibilities.add(columnVisibility); + if (expression != null) { + this.accessExpressions.add(expression); } } @@ -71,8 +71,8 @@ public AggregateOperation getOperation() { } @Override - public Set getColumnVisibilities() { - return Collections.unmodifiableSet(columnVisibilities); + public Set getAccessExpressions() { + return Collections.unmodifiableSet(accessExpressions); } /** @@ -129,7 +129,7 @@ public void aggregate(Attribute value) { divisor = divisor.add(BigDecimal.ONE); } average = numerator.divide(divisor, MATH_CONTEXT); - columnVisibilities.add(value.getColumnVisibility()); + accessExpressions.add(value.getAccessExpression()); } @Override @@ -139,7 +139,7 @@ public void merge(Aggregator other) { this.numerator = numerator.add(aggregator.numerator); this.divisor = divisor.add(aggregator.divisor); this.average = this.numerator.divide(this.divisor, MATH_CONTEXT); - this.columnVisibilities.addAll(aggregator.columnVisibilities); + this.accessExpressions.addAll(aggregator.accessExpressions); } else { throw new IllegalArgumentException("Cannot merge instance of " + other.getClass().getName()); } @@ -148,6 +148,6 @@ public void merge(Aggregator other) { @Override public String toString() { return new ToStringBuilder(this).append("field", field).append("average", average).append("numerator", numerator).append("divisor", divisor) - .append("columnVisibilities", columnVisibilities).toString(); + .append("accessExpressions", accessExpressions).toString(); } } diff --git a/warehouse/query-core/src/main/java/datawave/query/common/grouping/CountAggregator.java b/warehouse/query-core/src/main/java/datawave/query/common/grouping/CountAggregator.java index 1144d1f298f..dc4c4243c3f 100644 --- a/warehouse/query-core/src/main/java/datawave/query/common/grouping/CountAggregator.java +++ b/warehouse/query-core/src/main/java/datawave/query/common/grouping/CountAggregator.java @@ -5,7 +5,7 @@ import java.util.HashSet; import java.util.Set; -import org.apache.accumulo.core.security.ColumnVisibility; +import org.apache.accumulo.access.AccessExpression; import org.apache.commons.lang.builder.ToStringBuilder; import datawave.query.attributes.Attribute; @@ -22,24 +22,24 @@ public class CountAggregator extends AbstractAggregator { private long count; /** - * The column visibilities of all attributes aggregated. + * The access expressions of all attributes aggregated. */ - private final Set columnVisibilities; + private final Set accessExpressions; public static CountAggregator of(String field, TypeAttribute attribute) { - return new CountAggregator(field, attribute.getType().getDelegate().longValue(), attribute.getColumnVisibility()); + return new CountAggregator(field, attribute.getType().getDelegate().longValue(), attribute.getAccessExpression()); } public CountAggregator(String field) { super(field); - this.columnVisibilities = new HashSet<>(); + this.accessExpressions = new HashSet<>(); } - private CountAggregator(String field, long count, ColumnVisibility visibility) { + private CountAggregator(String field, long count, AccessExpression expression) { this(field); this.count = count; - if (visibility != null) { - columnVisibilities.add(visibility); + if (expression != null) { + accessExpressions.add(expression); } } @@ -54,8 +54,8 @@ public AggregateOperation getOperation() { } @Override - public Set getColumnVisibilities() { - return Collections.unmodifiableSet(columnVisibilities); + public Set getAccessExpressions() { + return Collections.unmodifiableSet(accessExpressions); } /** @@ -82,7 +82,7 @@ public boolean hasAggregation() { @Override public void aggregate(Attribute value) { count++; - this.columnVisibilities.add(value.getColumnVisibility()); + this.accessExpressions.add(value.getAccessExpression()); } @Override @@ -90,7 +90,7 @@ public void merge(Aggregator other) { if (other instanceof CountAggregator) { CountAggregator aggregator = (CountAggregator) other; this.count += aggregator.count; - this.columnVisibilities.addAll(aggregator.columnVisibilities); + this.accessExpressions.addAll(aggregator.accessExpressions); } else { throw new IllegalArgumentException("Cannot merge instance of " + other.getClass().getName()); } @@ -98,6 +98,6 @@ public void merge(Aggregator other) { @Override public String toString() { - return new ToStringBuilder(this).append("field", field).append("count", count).append("columnVisibilities", columnVisibilities).toString(); + return new ToStringBuilder(this).append("field", field).append("count", count).append("accessExpressions", accessExpressions).toString(); } } diff --git a/warehouse/query-core/src/main/java/datawave/query/common/grouping/DocumentGrouper.java b/warehouse/query-core/src/main/java/datawave/query/common/grouping/DocumentGrouper.java index b7131c34b65..838a16e71df 100644 --- a/warehouse/query-core/src/main/java/datawave/query/common/grouping/DocumentGrouper.java +++ b/warehouse/query-core/src/main/java/datawave/query/common/grouping/DocumentGrouper.java @@ -310,7 +310,7 @@ private void extractGroupsFromDocument() { // Create a new group and merge it into the existing groups. Group group = new Group(grouping, count); group.setFieldAggregator(fieldAggregator); - group.addDocumentVisibility(document.getColumnVisibility()); + group.addDocumentExpression(document.getAccessExpression()); groups.mergeOrPutGroup(group); } } @@ -630,9 +630,9 @@ private void trackGroup(Grouping grouping) { } // Add the visibilities of each attribute in the grouping for combination later, and increment the count for how many times this distinct // grouping was seen. - group.addAttributeVisibilities(grouping); + group.addAttributeExpressions(grouping); group.incrementCount(); - group.addDocumentVisibility(document.getColumnVisibility()); + group.addDocumentExpression(document.getAccessExpression()); } private Set> createGroupingAttributes(String field, Set> attributes) { diff --git a/warehouse/query-core/src/main/java/datawave/query/common/grouping/Group.java b/warehouse/query-core/src/main/java/datawave/query/common/grouping/Group.java index d6f1388ee34..1173042745b 100644 --- a/warehouse/query-core/src/main/java/datawave/query/common/grouping/Group.java +++ b/warehouse/query-core/src/main/java/datawave/query/common/grouping/Group.java @@ -3,13 +3,17 @@ import java.util.Collection; import java.util.HashSet; import java.util.Set; +import java.util.stream.Collectors; +import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.commons.lang.builder.ToStringBuilder; import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; +import datawave.marking.AccessExpressionUtil; + /** * Represents a grouping of values for fields specified via the #GROUP_BY functionality, with information about the total number of times the grouping was seen, * values for target aggregation fields that were matched to this group, and the different column visibilities seen. @@ -22,14 +26,14 @@ public class Group { private final ImmutableGrouping grouping; /** - * The different column visibilities seen for each attribute that makes up the grouping. + * The different access expressions seen for each attribute that makes up the grouping. */ - private final Multimap,ColumnVisibility> attributeVisibilities = HashMultimap.create(); + private final Multimap,AccessExpression> attributeExpressions = HashMultimap.create(); /** - * The column visibilities for each document that contributed entries to this grouping. + * The access expressions for each document that contributed entries to this grouping. */ - private final Set documentVisibilities = new HashSet<>(); + private final Set documentExpressions = new HashSet<>(); /** * The total number of times the distinct grouping was seen. @@ -47,7 +51,7 @@ public Group(Grouping grouping) { public Group(Grouping grouping, int count) { this.grouping = new ImmutableGrouping(grouping); - addAttributeVisibilities(this.grouping); + addAttributeExpressions(this.grouping); this.count = count; } @@ -70,45 +74,65 @@ public Grouping getGrouping() { } /** - * Add the column visibilities from each of the given attributes to the set of attribute visibilities for this group. + * Add the access expressions from each of the given attributes to the set of attribute expressions for this group. * * @param grouping - * the attributes to add visibilities from + * the attributes to add expressions from */ - public void addAttributeVisibilities(Grouping grouping) { + public void addAttributeExpressions(Grouping grouping) { for (GroupingAttribute attribute : grouping) { - attributeVisibilities.put(attribute, attribute.getColumnVisibility()); + attributeExpressions.put(attribute, attribute.getAccessExpression()); } } /** - * Return the set of column visibilities seen for the given attribute. + * Return the set of visibilities seen for the given attribute. * * @param attribute * the attribute - * @return the column visibilities seen for the given attributes + * @return the access expressions converted to ColumnVisibilities seen for the given attribute + */ + public Collection getColumnVisibilitiesForAttribute(GroupingAttribute attribute) { + return attributeExpressions.get(attribute).stream().map(AccessExpressionUtil::toColumnVisibility).collect(Collectors.toSet()); + } + + /** + * Return the set of access expressions seen for the given attribute. + * + * @param attribute + * the attribute + * @return the access expressions seen for the given attribute + */ + public Collection getAccessExpressionsForAttribute(GroupingAttribute attribute) { + return attributeExpressions.get(attribute); + } + + /** + * Add the access expression to the set of expressions of documents for which we have seen the grouping of this group in. + * + * @param expression + * the expression to add */ - public Collection getVisibilitiesForAttribute(GroupingAttribute attribute) { - return attributeVisibilities.get(attribute); + public void addDocumentExpression(AccessExpression expression) { + this.documentExpressions.add(expression); } /** - * Add the column visibility to the set of visibilities of documents for which we have seen the grouping of this group in. + * Return the set of all distinct access expressions from documents that we have seen this group in. * - * @param columnVisibility - * the visibility to add + * @return the document access expressions */ - public void addDocumentVisibility(ColumnVisibility columnVisibility) { - this.documentVisibilities.add(columnVisibility); + public Set getDocumentExpressions() { + return documentExpressions; } /** - * Return the set of all distinct column visibilities from documents that we have seen this group in. + * Return the set of all distinct visibilities from documents that we have seen this group in. * - * @return the document column visibilities + * @return the document access expressions converted into ColumnVisibilities */ public Set getDocumentVisibilities() { - return documentVisibilities; + return documentExpressions.stream().map(AccessExpressionUtil::toColumnVisibility).collect(Collectors.toSet()); } /** @@ -163,15 +187,15 @@ public void aggregateAll(String field, Collection fields) { * the group to merge */ public void merge(Group other) { - this.attributeVisibilities.putAll(other.attributeVisibilities); - this.documentVisibilities.addAll(other.documentVisibilities); + this.attributeExpressions.putAll(other.attributeExpressions); + this.documentExpressions.addAll(other.documentExpressions); this.count += other.count; this.fieldAggregator.merge(other.fieldAggregator); } @Override public String toString() { - return new ToStringBuilder(this).append("grouping", grouping).append("attributeVisibilities", attributeVisibilities) - .append("documentVisibilities", documentVisibilities).append("count", count).append("aggregatedFields", fieldAggregator).toString(); + return new ToStringBuilder(this).append("grouping", grouping).append("attributeExpressions", attributeExpressions) + .append("documentExpressions", documentExpressions).append("count", count).append("aggregatedFields", fieldAggregator).toString(); } } diff --git a/warehouse/query-core/src/main/java/datawave/query/common/grouping/GroupingUtils.java b/warehouse/query-core/src/main/java/datawave/query/common/grouping/GroupingUtils.java index 34375910c4a..676a26efcb3 100644 --- a/warehouse/query-core/src/main/java/datawave/query/common/grouping/GroupingUtils.java +++ b/warehouse/query-core/src/main/java/datawave/query/common/grouping/GroupingUtils.java @@ -7,7 +7,9 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; +import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.security.ColumnVisibility; import org.slf4j.Logger; @@ -15,6 +17,7 @@ import com.google.common.base.Preconditions; import datawave.data.type.NumberType; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.query.attributes.Document; import datawave.query.attributes.TypeAttribute; @@ -31,25 +34,25 @@ public enum AverageAggregatorWriteFormat { private static final Logger log = getLogger(GroupingUtils.class); /** - * Returns a column visibility that results from the combination of all given visibilities using the given {@link MarkingFunctions}. + * Returns a column visibility that results from the combination of all given access expressions using the given {@link MarkingFunctions}. * - * @param visibilities - * the visibilities to combine + * @param expressions + * the access expressions to combine * @param markingFunctions - * the marking functions to combine the visibilities with + * the marking functions to combine the expressions with * @param failOnError - * if true and the visibilities cannot be combined, an {@link IllegalArgumentException} will be thrown. If false and the visibilities cannot be + * if true and the expressions cannot be combined, an {@link IllegalArgumentException} will be thrown. If false and the expressions cannot be * combined, it will be logged and a new, blank {@link ColumnVisibility} will be returned. * @return the combined column visibility */ - public static ColumnVisibility combineVisibilities(Collection visibilities, MarkingFunctions markingFunctions, boolean failOnError) { + public static ColumnVisibility combineVisibilities(Collection expressions, MarkingFunctions markingFunctions, boolean failOnError) { try { - return markingFunctions.combine(visibilities); + return markingFunctions.combineVisibilities(expressions.stream().map(AccessExpressionUtil::toColumnVisibility).collect(Collectors.toList())); } catch (MarkingFunctions.Exception e) { if (failOnError) { - throw new IllegalArgumentException("Unable to combine visibilities: " + visibilities, e); + throw new IllegalArgumentException("Unable to combine access expressions: " + expressions, e); } else { - log.warn("Unable to combine visibilities from {}", visibilities); + log.warn("Unable to combine access expressions from {}", expressions); } } return new ColumnVisibility(); @@ -71,7 +74,7 @@ public static ColumnVisibility combineVisibilities(Collection * @return the new document */ @Deprecated - public static Document createDocument(Group group, List keys, MarkingFunctions markingFunctions, AverageAggregatorWriteFormat averageWriteFormat) { + public static Document createDocument(Group group, List keys, MarkingFunctions markingFunctions, AverageAggregatorWriteFormat averageWriteFormat) { Preconditions.checkState(!keys.isEmpty(), "No available keys for grouping results"); // Use the last (most recent) key so a new iterator will know where to start. @@ -92,17 +95,17 @@ public static Document createDocument(Group group, List keys, MarkingFuncti * the format to use when writing aggregated averages to the document * @return the new document */ - public static Document createDocument(Group group, Key key, MarkingFunctions markingFunctions, AverageAggregatorWriteFormat averageWriteFormat) { + public static Document createDocument(Group group, Key key, MarkingFunctions markingFunctions, AverageAggregatorWriteFormat averageWriteFormat) { Preconditions.checkNotNull(key, "document key cannot be null"); Document document = new Document(key, true); // Set the visibility for the document to the combined visibility of each previous document in which this grouping was seen in. - document.setColumnVisibility(combineVisibilities(group.getDocumentVisibilities(), markingFunctions, true)); + document.setColumnVisibility(combineVisibilities(group.getDocumentExpressions(), markingFunctions, true)); // Add each of the grouping attributes to the document. for (GroupingAttribute attribute : group.getGrouping()) { // Update the visibility to the combined visibilities of each visibility seen for this attribute in a grouping. - attribute.setColumnVisibility(combineVisibilities(group.getVisibilitiesForAttribute(attribute), markingFunctions, false)); + attribute.setColumnVisibility(combineVisibilities(group.getAccessExpressionsForAttribute(attribute), markingFunctions, false)); document.put(attribute.getMetadata().getRow().toString(), attribute); } @@ -167,11 +170,11 @@ public static Document createDocument(Group group, Key key, MarkingFunctions mar * @param markingFunctions * the marking functions to use when combining column visibilities */ - private static void addSumAggregation(Document document, String field, SumAggregator aggregator, MarkingFunctions markingFunctions) { + private static void addSumAggregation(Document document, String field, SumAggregator aggregator, MarkingFunctions markingFunctions) { NumberType type = new NumberType(); type.setDelegate(aggregator.getAggregation()); TypeAttribute sumAttribute = new TypeAttribute<>(type, new Key(field + "_sum"), true); - sumAttribute.setColumnVisibility(combineVisibilities(aggregator.getColumnVisibilities(), markingFunctions, false)); + sumAttribute.setColumnVisibility(combineVisibilities(aggregator.getAccessExpressions(), markingFunctions, false)); document.put(field + DocumentGrouper.FIELD_SUM_SUFFIX, sumAttribute); } @@ -187,13 +190,13 @@ private static void addSumAggregation(Document document, String field, SumAggreg * @param markingFunctions * the marking functions to use when combining column visibilities */ - private static void addCountAggregation(Document document, String field, CountAggregator aggregator, MarkingFunctions markingFunctions) { + private static void addCountAggregation(Document document, String field, CountAggregator aggregator, MarkingFunctions markingFunctions) { NumberType type = new NumberType(); type.setDelegate(BigDecimal.valueOf(aggregator.getAggregation())); TypeAttribute sumAttribute = new TypeAttribute<>(type, new Key(field + "_count"), true); - Set columnVisibilities = aggregator.getColumnVisibilities(); - if (!columnVisibilities.isEmpty()) { - sumAttribute.setColumnVisibility(combineVisibilities(aggregator.getColumnVisibilities(), markingFunctions, false)); + Set accessExpressions = aggregator.getAccessExpressions(); + if (!accessExpressions.isEmpty()) { + sumAttribute.setColumnVisibility(combineVisibilities(aggregator.getAccessExpressions(), markingFunctions, false)); } document.put(field + DocumentGrouper.FIELD_COUNT_SUFFIX, sumAttribute); } @@ -238,11 +241,11 @@ private static void addMaxAggregation(Document document, String field, MaxAggreg * @param markingFunctions * the marking functions to use when combining column visibilities */ - private static void addAverage(Document document, String field, AverageAggregator aggregator, MarkingFunctions markingFunctions) { + private static void addAverage(Document document, String field, AverageAggregator aggregator, MarkingFunctions markingFunctions) { NumberType type = new NumberType(); type.setDelegate(aggregator.getAggregation()); TypeAttribute attribute = new TypeAttribute<>(type, new Key(field + "_average"), true); - attribute.setColumnVisibility(combineVisibilities(aggregator.getColumnVisibilities(), markingFunctions, false)); + attribute.setColumnVisibility(combineVisibilities(aggregator.getAccessExpressions(), markingFunctions, false)); document.put(field + DocumentGrouper.FIELD_AVERAGE_SUFFIX, attribute); } @@ -258,8 +261,8 @@ private static void addAverage(Document document, String field, AverageAggregato * @param markingFunctions * the marking functions to use when combining column visibilities */ - private static void addAverageNumeratorAndDivisor(Document document, String field, AverageAggregator aggregator, MarkingFunctions markingFunctions) { - ColumnVisibility visibility = combineVisibilities(aggregator.getColumnVisibilities(), markingFunctions, false); + private static void addAverageNumeratorAndDivisor(Document document, String field, AverageAggregator aggregator, MarkingFunctions markingFunctions) { + ColumnVisibility visibility = combineVisibilities(aggregator.getAccessExpressions(), markingFunctions, false); // Add an attribute for the average's numerator. This is required to properly combine additional aggregations in future groupings. NumberType numeratorType = new NumberType(); diff --git a/warehouse/query-core/src/main/java/datawave/query/common/grouping/MaxAggregator.java b/warehouse/query-core/src/main/java/datawave/query/common/grouping/MaxAggregator.java index da9f538040e..435142c876e 100644 --- a/warehouse/query-core/src/main/java/datawave/query/common/grouping/MaxAggregator.java +++ b/warehouse/query-core/src/main/java/datawave/query/common/grouping/MaxAggregator.java @@ -3,7 +3,7 @@ import java.util.Collections; import java.util.Set; -import org.apache.accumulo.core.security.ColumnVisibility; +import org.apache.accumulo.access.AccessExpression; import org.apache.commons.lang.builder.ToStringBuilder; import datawave.query.attributes.Attribute; @@ -40,14 +40,14 @@ public AggregateOperation getOperation() { } /** - * Returns a singleton set containing the column visibility of the max attribute found. Possible empty, but never null. + * Returns a singleton set containing the access expression of the max attribute found. Possibly empty, but never null. * - * @return a set containing the column visibility + * @return a set containing the access expression */ @Override - public Set getColumnVisibilities() { + public Set getAccessExpressions() { if (max != null) { - return Collections.singleton(max.getColumnVisibility()); + return Collections.singleton(max.getAccessExpression()); } return Collections.emptySet(); } diff --git a/warehouse/query-core/src/main/java/datawave/query/common/grouping/MinAggregator.java b/warehouse/query-core/src/main/java/datawave/query/common/grouping/MinAggregator.java index 2a8d4a17316..7f652d0ba64 100644 --- a/warehouse/query-core/src/main/java/datawave/query/common/grouping/MinAggregator.java +++ b/warehouse/query-core/src/main/java/datawave/query/common/grouping/MinAggregator.java @@ -3,7 +3,7 @@ import java.util.Collections; import java.util.Set; -import org.apache.accumulo.core.security.ColumnVisibility; +import org.apache.accumulo.access.AccessExpression; import org.apache.commons.lang.builder.ToStringBuilder; import datawave.query.attributes.Attribute; @@ -43,14 +43,14 @@ public AggregateOperation getOperation() { } /** - * Returns a singleton set containing the column visibility of the min attribute found. Possible empty, but never null. + * Returns a singleton set containing the access expression of the min attribute found. Possibly empty, but never null. * - * @return a set containing the column visibility + * @return a set containing the access expression */ @Override - public Set getColumnVisibilities() { + public Set getAccessExpressions() { if (min != null) { - return Collections.singleton(min.getColumnVisibility()); + return Collections.singleton(min.getAccessExpression()); } return Collections.emptySet(); } diff --git a/warehouse/query-core/src/main/java/datawave/query/common/grouping/SumAggregator.java b/warehouse/query-core/src/main/java/datawave/query/common/grouping/SumAggregator.java index 888544af9fe..0e79304c5a3 100644 --- a/warehouse/query-core/src/main/java/datawave/query/common/grouping/SumAggregator.java +++ b/warehouse/query-core/src/main/java/datawave/query/common/grouping/SumAggregator.java @@ -5,7 +5,7 @@ import java.util.HashSet; import java.util.Set; -import org.apache.accumulo.core.security.ColumnVisibility; +import org.apache.accumulo.access.AccessExpression; import org.apache.commons.lang.builder.ToStringBuilder; import datawave.query.attributes.Attribute; @@ -23,25 +23,25 @@ public class SumAggregator extends AbstractAggregator { private BigDecimal sum; /** - * The column visibilities of all attributes aggregated. + * The access expressions of all attributes aggregated. */ - private final Set columnVisibilities; + private final Set accessExpressions; public static SumAggregator of(String field, TypeAttribute attribute) { BigDecimal sum = attribute.getType().getDelegate(); - return new SumAggregator(field, sum, attribute.getColumnVisibility()); + return new SumAggregator(field, sum, attribute.getAccessExpression()); } public SumAggregator(String field) { super(field); - this.columnVisibilities = new HashSet<>(); + this.accessExpressions = new HashSet<>(); } - private SumAggregator(String field, BigDecimal sum, ColumnVisibility visibility) { + private SumAggregator(String field, BigDecimal sum, AccessExpression expression) { this(field); this.sum = sum; - if (visibility != null) { - this.columnVisibilities.add(visibility); + if (expression != null) { + this.accessExpressions.add(expression); } } @@ -56,8 +56,8 @@ public AggregateOperation getOperation() { } @Override - public Set getColumnVisibilities() { - return Collections.unmodifiableSet(columnVisibilities); + public Set getAccessExpressions() { + return Collections.unmodifiableSet(accessExpressions); } /** @@ -96,7 +96,7 @@ public void aggregate(Attribute value) { } else { sum = sum.add(number); } - columnVisibilities.add(value.getColumnVisibility()); + accessExpressions.add(value.getAccessExpression()); } @Override @@ -104,7 +104,7 @@ public void merge(Aggregator other) { if (other instanceof SumAggregator) { SumAggregator aggregator = (SumAggregator) other; this.sum = this.sum.add(aggregator.sum); - this.columnVisibilities.addAll(aggregator.columnVisibilities); + this.accessExpressions.addAll(aggregator.accessExpressions); } else { throw new IllegalArgumentException("Cannot merge instance of " + other.getAggregation()); } @@ -112,6 +112,6 @@ public void merge(Aggregator other) { @Override public String toString() { - return new ToStringBuilder(this).append("field", field).append("sum", sum).append("columnVisibilities", columnVisibilities).toString(); + return new ToStringBuilder(this).append("field", field).append("sum", sum).append("accessExpressions", accessExpressions).toString(); } } diff --git a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java index 404d9c29dda..ef95c3cc452 100644 --- a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java +++ b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java @@ -31,13 +31,13 @@ import com.google.protobuf.InvalidProtocolBufferException; import datawave.ingest.protobuf.Uid; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.query.Constants; public class DiscoveryIterator implements SortedKeyValueIterator { private static final Logger log = Logger.getLogger(DiscoveryIterator.class); - private static final MarkingFunctions markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); private Key key; private Value value; @@ -46,6 +46,7 @@ public class DiscoveryIterator implements SortedKeyValueIterator { private boolean showReferenceCount = false; private boolean reverseIndex = false; private boolean sumCounts = false; + private MarkingFunctions markingFunctions; @Override public DiscoveryIterator deepCopy(IteratorEnvironment env) { @@ -162,10 +163,15 @@ private DiscoveredThing aggregate(Collection termEntries) { } else { // Otherwise, combine the visibilities, and return the aggregated result. try { - ColumnVisibility visibility = markingFunctions.combine(visibilities); + if (null == markingFunctions) { + markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); + } + ColumnVisibility visibility = markingFunctions.combineVisibilities(visibilities); + MapWritable countsByVis = new MapWritable(); visibilityToCounts.forEach((key, value) -> countsByVis.put(new Text(key), new LongWritable(value))); - return new DiscoveredThing(term, first.getField(), first.getDatatype(), date, new String(visibility.flatten()), count, countsByVis); + String normalizedVis = AccessExpressionUtil.normalize(visibility).getExpression(); + return new DiscoveredThing(term, first.getField(), first.getDatatype(), date, normalizedVis, count, countsByVis); } catch (Exception e) { if (log.isTraceEnabled()) { log.warn("Invalid column visibilities after combining " + visibilities); diff --git a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryTransformer.java index bbabc7c9537..f1ca542b321 100644 --- a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryTransformer.java @@ -1,7 +1,6 @@ package datawave.query.discovery; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -14,7 +13,7 @@ import datawave.core.query.logic.BaseQueryLogic; import datawave.core.query.logic.BaseQueryLogicTransformer; import datawave.marking.MarkingFunctions; -import datawave.marking.MarkingFunctions.Exception; +import datawave.marking.Markings; import datawave.microservice.query.Query; import datawave.query.model.QueryModel; import datawave.webservice.query.cachedresults.CacheableQueryRow; @@ -30,7 +29,7 @@ public class DiscoveryTransformer extends BaseQueryLogicTransformer variableFieldList = null; private BaseQueryLogic logic = null; private QueryModel myQueryModel = null; - private MarkingFunctions markingFunctions; + private MarkingFunctions markingFunctions; private ResponseObjectFactory responseObjectFactory; public DiscoveryTransformer(BaseQueryLogic logic, Query settings, QueryModel qm) { @@ -46,10 +45,10 @@ public EventBase transform(DiscoveredThing thing) { Preconditions.checkNotNull(thing, "Received a null object to transform!"); EventBase event = this.responseObjectFactory.getEvent(); - Map markings; + Markings markings; try { - markings = this.markingFunctions.translateFromColumnVisibility(new ColumnVisibility(thing.getColumnVisibility())); - } catch (Exception e) { + markings = markingFunctions.translateFromColumnVisibility(new ColumnVisibility(thing.getColumnVisibility())); + } catch (MarkingFunctions.Exception e) { throw new RuntimeException("could not parse to markings: " + thing.getColumnVisibility()); } event.setMarkings(markings); @@ -67,12 +66,13 @@ public EventBase transform(DiscoveredThing thing) { // If requested return counts separated by colvis, all counts by colvis could be > total record count if (thing.getCountsByColumnVisibility() != null && !thing.getCountsByColumnVisibility().isEmpty()) { for (Map.Entry entry : thing.getCountsByColumnVisibility().entrySet()) { + Markings recordCountMarkings; try { - Map eMarkings = this.markingFunctions.translateFromColumnVisibility(new ColumnVisibility(entry.getKey().toString())); - fields.add(this.makeField("RECORD COUNT", new HashMap<>(), entry.getKey().toString(), 0L, entry.getValue().toString())); - } catch (Exception e) { + recordCountMarkings = markingFunctions.translateFromColumnVisibility(new ColumnVisibility(entry.getKey().toString())); + } catch (MarkingFunctions.Exception e) { throw new RuntimeException("could not parse to markings: " + thing.getColumnVisibility()); } + fields.add(this.makeField("RECORD COUNT", recordCountMarkings, entry.getKey().toString(), 0L, entry.getValue().toString())); } } else { @@ -92,7 +92,7 @@ public EventBase transform(DiscoveredThing thing) { return event; } - protected FieldBase makeField(String name, Map markings, String columnVisibility, Long timestamp, Object value) { + protected FieldBase makeField(String name, Markings markings, String columnVisibility, Long timestamp, Object value) { FieldBase field = this.responseObjectFactory.getField(); field.setName(name); field.setMarkings(markings); @@ -140,7 +140,7 @@ public Object readFromCache(CacheableQueryRow cacheableQueryRow) { if (this.variableFieldList == null) { this.variableFieldList = cacheableQueryRow.getVariableColumnNames(); } - Map markings = cacheableQueryRow.getMarkings(); + Markings markings = cacheableQueryRow.getMarkings(); String dataType = cacheableQueryRow.getDataType(); String internalId = cacheableQueryRow.getEventId(); String row = cacheableQueryRow.getRow(); @@ -163,7 +163,7 @@ public Object readFromCache(CacheableQueryRow cacheableQueryRow) { String columnValue = entry.getValue(); String columnVisibility = cacheableQueryRow.getColumnVisibility(columnName); Long columnTimestamp = cacheableQueryRow.getColumnTimestamp(columnName); - Map columnMarkings = cacheableQueryRow.getColumnMarkings(columnName); + Markings columnMarkings = cacheableQueryRow.getColumnMarkings(columnName); FieldBase field = this.responseObjectFactory.getField(); field.setName(columnName); field.setMarkings(columnMarkings); diff --git a/warehouse/query-core/src/main/java/datawave/query/discovery/TermInfoAggregation.java b/warehouse/query-core/src/main/java/datawave/query/discovery/TermInfoAggregation.java index 960a83e1889..8239a5e643b 100644 --- a/warehouse/query-core/src/main/java/datawave/query/discovery/TermInfoAggregation.java +++ b/warehouse/query-core/src/main/java/datawave/query/discovery/TermInfoAggregation.java @@ -15,6 +15,7 @@ import com.google.common.base.Function; import com.google.common.collect.Sets; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; public class TermInfoAggregation implements Function,DiscoveredThing> { @@ -24,6 +25,7 @@ public class TermInfoAggregation implements Function,Discov private final boolean separateCountsByColumnVisibility; private boolean showReferenceCountInsteadOfTermCount = false; private boolean reverseIndex = false; + private MarkingFunctions markingFunctions; public TermInfoAggregation() { this.separateCountsByColumnVisibility = false; @@ -73,7 +75,7 @@ public DiscoveredThing apply(Collection from) { chosenCount = showReferenceCountInsteadOfTermCount ? referenceCount : termCount; try { - MarkingFunctions.Factory.createMarkingFunctions().translateFromColumnVisibility(ti.vis); // just to test parsing + markingFunctions.translateFromColumnVisibility(ti.vis); // just to test parsing columnVisibilities.add(ti.vis); // Keep track of counts for individual vis @@ -104,11 +106,12 @@ public DiscoveredThing apply(Collection from) { log.trace("Did not aggregate any counts for [" + term + "][" + field + "][" + type + "][" + date + "]. Returning null."); return null; } else { - ColumnVisibility columnVisibility = null; + ColumnVisibility columnVisibility; try { - - columnVisibility = MarkingFunctions.Factory.createMarkingFunctions().combine(columnVisibilities); - + if (null == markingFunctions) { + markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); + } + columnVisibility = markingFunctions.combineVisibilities(columnVisibilities); } catch (Exception e) { log.warn("Invalid columnvisibility after combining!", e); return null; @@ -119,7 +122,7 @@ public DiscoveredThing apply(Collection from) { countsByVis.put(new Text(entry.getKey()), new VLongWritable(entry.getValue())); } - return new DiscoveredThing(term, field, type, date, new String(columnVisibility.flatten()), count, countsByVis); + return new DiscoveredThing(term, field, type, date, AccessExpressionUtil.normalize(columnVisibility).getExpression(), count, countsByVis); } } } diff --git a/warehouse/query-core/src/main/java/datawave/query/iterator/GroupingIterator.java b/warehouse/query-core/src/main/java/datawave/query/iterator/GroupingIterator.java index 43928b68b77..ae0690e98c2 100644 --- a/warehouse/query-core/src/main/java/datawave/query/iterator/GroupingIterator.java +++ b/warehouse/query-core/src/main/java/datawave/query/iterator/GroupingIterator.java @@ -10,9 +10,9 @@ import java.util.Map; import java.util.Set; +import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.iterators.YieldCallback; -import org.apache.accumulo.core.security.ColumnVisibility; import org.slf4j.Logger; import com.google.common.collect.Maps; @@ -51,7 +51,7 @@ public class GroupingIterator implements Iterator> { // track the number of documents seen by this iterator private long documentCount = 0L; - private final MarkingFunctions markingFunctions; + private final MarkingFunctions markingFunctions; private final int groupFieldsBatchSize; @@ -61,7 +61,7 @@ public class GroupingIterator implements Iterator> { Map.Entry next; - public GroupingIterator(Iterator> previousIterators, MarkingFunctions markingFunctions, GroupFields groupFields, + public GroupingIterator(Iterator> previousIterators, MarkingFunctions markingFunctions, GroupFields groupFields, int groupFieldsBatchSize, YieldCallback yieldCallback) { this.previousIterators = previousIterators; this.markingFunctions = markingFunctions; @@ -187,11 +187,11 @@ private Document flatten(List documents) { Document flattened = new Document(documents.get(documents.size() - 1).getMetadata(), true); int context = 0; - Set visibilities = new HashSet<>(); + Set expressions = new HashSet<>(); for (Document document : documents) { log.trace("document: {}", document); for (Map.Entry>> entry : document.entrySet()) { - visibilities.add(entry.getValue().getColumnVisibility()); + expressions.add(entry.getValue().getAccessExpression()); // Add a copy of each attribute to the flattened document with the context appended to the key, e.g. AGE becomes AGE.0. Attribute> attribute = entry.getValue(); attribute.setColumnVisibility(entry.getValue().getColumnVisibility()); @@ -203,8 +203,8 @@ private Document flatten(List documents) { context++; } - // Set the flattened document's visibility to the combined visibilities of each document. - flattened.setColumnVisibility(GroupingUtils.combineVisibilities(visibilities, markingFunctions, false)); + // Set the flattened document's visibility to the combined access expressions of each document. + flattened.setColumnVisibility(GroupingUtils.combineVisibilities(expressions, markingFunctions, false)); log.trace("flattened document: {}", flattened); return flattened; } diff --git a/warehouse/query-core/src/main/java/datawave/query/iterators/FieldIndexCountingIterator.java b/warehouse/query-core/src/main/java/datawave/query/iterators/FieldIndexCountingIterator.java index d6f11096c20..0cfa40e3642 100644 --- a/warehouse/query-core/src/main/java/datawave/query/iterators/FieldIndexCountingIterator.java +++ b/warehouse/query-core/src/main/java/datawave/query/iterators/FieldIndexCountingIterator.java @@ -91,6 +91,7 @@ public class FieldIndexCountingIterator extends WrappingIterator implements Sort public static final Text fi_PREFIX_TEXT = new Text("fi\u0000"); private Set visibilitySet = new HashSet<>(); + private MarkingFunctions markingFunctions; // ------------------------------------------------------------------------- // ------------- Constructors @@ -585,15 +586,18 @@ private Key buildReturnKey() { for (Text t : this.visibilitySet) { columnVisibilities.add(new ColumnVisibility(t)); } - ColumnVisibility cv; + ColumnVisibility columnVisibility; try { - cv = MarkingFunctions.Factory.createMarkingFunctions().combine(columnVisibilities); - } catch (MarkingFunctions.Exception e) { - log.error("Could not combine visibilities: " + visibilitySet + " " + e); + if (null == markingFunctions) { + markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); + } + columnVisibility = markingFunctions.combineVisibilities(columnVisibilities); + } catch (Exception e) { + log.warn("Invalid columnvisibility after combining!", e); return null; } - return new Key(this.currentRow, new Text(this.currentFieldName), cq, new Text(cv.getExpression()), this.maxTimeStamp); + return new Key(this.currentRow, new Text(this.currentFieldName), cq, new Text(columnVisibility.getExpression()), this.maxTimeStamp); } /* TODO: make this a mutable long, also check wrap up current method */ diff --git a/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java b/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java index 7cba3801a7f..66c863a308a 100644 --- a/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java +++ b/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java @@ -45,7 +45,6 @@ import datawave.core.iterators.DatawaveFieldIndexListIteratorJexl; import datawave.marking.MarkingFunctions; -import datawave.marking.MarkingFunctionsFactory; import datawave.query.attributes.Attribute; import datawave.query.attributes.Attributes; import datawave.query.attributes.ValueTuple; @@ -71,6 +70,7 @@ public class DatawaveInterpreter extends Interpreter { protected Map resultMap; + private MarkingFunctions markingFunctions; private static final Logger log = Logger.getLogger(DatawaveInterpreter.class); @@ -452,7 +452,7 @@ private void addHitsForFunction(Object o, ASTFunctionNode node) { private void addHitsForFunction(String field, ASTFunctionNode node, HitListArithmetic hitListArithmetic) { ColumnVisibility cv; // aggregate individual hits for the content function - Collection cvs = new HashSet<>(); + Collection columnVisibilities = new HashSet<>(); Attributes source = new Attributes(true); ContentFunctionsDescriptor.ContentJexlArgumentDescriptor jexlArgDescriptor = new ContentFunctionsDescriptor().getArgumentDescriptor(node); @@ -463,19 +463,24 @@ private void addHitsForFunction(String field, ASTFunctionNode node, HitListArith if (values.contains(tuple.getNormalizedValue())) { Attribute attr = tuple.getSource(); source.add(attr); - cvs.add(attr.getColumnVisibility()); + columnVisibilities.add(attr.getColumnVisibility()); } } } + if (null == markingFunctions) { + markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); + } + ColumnVisibility columnVisibility; try { - cv = MarkingFunctionsFactory.createMarkingFunctions().combine(cvs); + columnVisibility = markingFunctions.combineVisibilities(columnVisibilities); } catch (MarkingFunctions.Exception e) { log.error("Failed to combine column visibilities while generating HIT_TERM for phrase function for field [" + field + "]"); log.error("msg: ", e); return; } - source.setColumnVisibility(cv); + + source.setColumnVisibility(columnVisibility); // create an Attributes backed ValueTuple String phrase = jexlArgDescriptor.getHitTermValue(); diff --git a/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java b/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java index f7c95de1c41..dcc683af70c 100644 --- a/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java +++ b/warehouse/query-core/src/main/java/datawave/query/predicate/ValueToAttributes.java @@ -45,13 +45,13 @@ public class ValueToAttributes implements Function,Iterable> compositeToFieldMap; private Map> compositeFieldSeparatorsByType; - private final MarkingFunctions markingFunctions; private final Multimap> componentFieldToValues = ArrayListMultimap.create(); private final EventDataQueryFilter attrFilter; // Whether the value is from the index private final boolean fromIndex; + private MarkingFunctions markingFunctions; /** * Constructor that accepts raw components @@ -67,8 +67,8 @@ public class ValueToAttributes implements Function,Iterable markingFunctions, boolean fromIndex) { // in Java 25 we can create the attribute factory first this(new AttributeFactory(typeMetadata), compositeMetadata, attrFilter, markingFunctions, fromIndex); } @@ -88,7 +88,7 @@ public ValueToAttributes(CompositeMetadata compositeMetadata, TypeMetadata typeM * flag denoting if this class is operating on values from the field index */ public ValueToAttributes(AttributeFactory attributeFactory, CompositeMetadata compositeMetadata, EventDataQueryFilter attrFilter, - MarkingFunctions markingFunctions, boolean fromIndex) { + MarkingFunctions markingFunctions, boolean fromIndex) { this.attrFactory = attributeFactory; this.markingFunctions = markingFunctions; this.attrFilter = attrFilter; @@ -290,8 +290,18 @@ public Attribute joinAttributes(String compositeName, Collection if (log.isDebugEnabled()) { log.debug("dataList is " + dataList); } - ColumnVisibility combinedColumnVisibility = this.markingFunctions.combine(columnVisibilities); - metadata = new Key(metadata.getRow(), metadata.getColumnFamily(), new Text(), combinedColumnVisibility, timestamp); + ColumnVisibility columnVisibility; + try { + if (null == markingFunctions) { + markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); + } + columnVisibility = markingFunctions.combineVisibilities(columnVisibilities); + } catch (java.lang.Exception e) { + log.warn("Invalid column visibility after combining!", e); + return null; + } + + metadata = new Key(metadata.getRow(), metadata.getColumnFamily(), new Text(), columnVisibility, timestamp); if (dataList.size() == 1) { return this.attrFactory.create(compositeName, dataList.get(0), metadata, toKeep, true); } else { diff --git a/warehouse/query-core/src/main/java/datawave/query/table/parser/ContentKeyValueFactory.java b/warehouse/query-core/src/main/java/datawave/query/table/parser/ContentKeyValueFactory.java index 8ed69bf246d..d2de6672545 100644 --- a/warehouse/query-core/src/main/java/datawave/query/table/parser/ContentKeyValueFactory.java +++ b/warehouse/query-core/src/main/java/datawave/query/table/parser/ContentKeyValueFactory.java @@ -19,7 +19,7 @@ public class ContentKeyValueFactory { private static final Logger log = Logger.getLogger(ContentKeyValueFactory.class); - public static ContentKeyValue parse(Key key, Value value, Authorizations auths, MarkingFunctions markingFunctions) throws MarkingFunctions.Exception { + public static ContentKeyValue parse(Key key, Value value, Authorizations auths, MarkingFunctions markingFunctions) throws MarkingFunctions.Exception { if (null == key) throw new IllegalArgumentException("Cannot pass null key to ContentKeyValueFactory"); diff --git a/warehouse/query-core/src/main/java/datawave/query/table/parser/EventKeyValueFactory.java b/warehouse/query-core/src/main/java/datawave/query/table/parser/EventKeyValueFactory.java index 60693cdefce..78a94ce0aa6 100644 --- a/warehouse/query-core/src/main/java/datawave/query/table/parser/EventKeyValueFactory.java +++ b/warehouse/query-core/src/main/java/datawave/query/table/parser/EventKeyValueFactory.java @@ -1,21 +1,17 @@ package datawave.query.table.parser; -import java.util.HashMap; -import java.util.Map; - import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.ColumnVisibility; -import com.google.common.collect.Maps; - import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.query.Constants; public class EventKeyValueFactory { - public static EventKeyValue parse(Key key, Value value, Authorizations auths, MarkingFunctions markingFunctions) throws MarkingFunctions.Exception { + public static EventKeyValue parse(Key key, Value value, Authorizations auths, MarkingFunctions markingFunctions) throws MarkingFunctions.Exception { if (null == key) throw new IllegalArgumentException("Cannot pass null key to EventKeyValueFactory"); @@ -42,7 +38,7 @@ public static EventKeyValue parse(Key key, Value value, Authorizations auths, Ma return e; } - protected static void parseColumnVisibility(EventKeyValue event, Key key, Authorizations auths, MarkingFunctions markingFunctions) + protected static void parseColumnVisibility(EventKeyValue event, Key key, Authorizations auths, MarkingFunctions markingFunctions) throws MarkingFunctions.Exception { event.setMarkings(markingFunctions.translateFromColumnVisibilityForAuths(new ColumnVisibility(key.getColumnVisibility()), auths)); } @@ -53,7 +49,7 @@ public static class EventKeyValue { protected String uid = null; protected String fieldName = null; protected String fieldValue = null; - protected Map markings = null; + protected Markings markings = null; protected long timestamp = 0L; public String getShardId() { @@ -104,14 +100,12 @@ protected void setTimestamp(long timestamp) { this.timestamp = timestamp; } - public Map getMarkings() { - if (this.markings == null) - this.markings = Maps.newHashMap(); - return Maps.newHashMap(markings); + public Markings getMarkings() { + return this.markings; } - public void setMarkings(Map markings) { - this.markings = (markings == null ? new HashMap<>() : new HashMap<>(markings)); + public void setMarkings(Markings markings) { + this.markings = markings; } } diff --git a/warehouse/query-core/src/main/java/datawave/query/table/parser/KeywordKeyValueFactory.java b/warehouse/query-core/src/main/java/datawave/query/table/parser/KeywordKeyValueFactory.java index 71a3fe315e2..0ee5ff66b45 100644 --- a/warehouse/query-core/src/main/java/datawave/query/table/parser/KeywordKeyValueFactory.java +++ b/warehouse/query-core/src/main/java/datawave/query/table/parser/KeywordKeyValueFactory.java @@ -12,7 +12,7 @@ /** Parses results returned from the KeywordExtractingIterator. Expects 'd' column keys, and serialized bytes as the value */ public class KeywordKeyValueFactory { - public static KeywordKeyValue parse(Key key, Value value, Authorizations auths, MarkingFunctions markingFunctions) throws MarkingFunctions.Exception { + public static KeywordKeyValue parse(Key key, Value value, Authorizations auths, MarkingFunctions markingFunctions) throws MarkingFunctions.Exception { if (null == key) throw new IllegalArgumentException("Cannot pass null key to KeywordKeyValueFactory"); diff --git a/warehouse/query-core/src/main/java/datawave/query/table/parser/TermFrequencyKeyValueFactory.java b/warehouse/query-core/src/main/java/datawave/query/table/parser/TermFrequencyKeyValueFactory.java index 466839c795a..3582ff20fe5 100644 --- a/warehouse/query-core/src/main/java/datawave/query/table/parser/TermFrequencyKeyValueFactory.java +++ b/warehouse/query-core/src/main/java/datawave/query/table/parser/TermFrequencyKeyValueFactory.java @@ -15,7 +15,8 @@ public class TermFrequencyKeyValueFactory { - public static TermFrequencyKeyValue parse(Key key, Value value, Authorizations auths, MarkingFunctions markingFunctions) throws MarkingFunctions.Exception { + public static TermFrequencyKeyValue parse(Key key, Value value, Authorizations auths, MarkingFunctions markingFunctions) + throws MarkingFunctions.Exception { if (null == key) { throw new IllegalArgumentException("Cannot pass null key to TermFrequencyKeyValueFactory"); } @@ -54,7 +55,7 @@ public static TermFrequencyKeyValue parse(Key key, Value value, Authorizations a return t; } - protected static void parseColumnVisibility(TermFrequencyKeyValue tfkv, Key key, Authorizations auths, MarkingFunctions markingFunctions) + protected static void parseColumnVisibility(TermFrequencyKeyValue tfkv, Key key, Authorizations auths, MarkingFunctions markingFunctions) throws MarkingFunctions.Exception { tfkv.setMarkings(markingFunctions.translateFromColumnVisibilityForAuths(key.getColumnVisibilityParsed(), auths)); } diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/BaseRemoteQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/BaseRemoteQueryLogic.java index 2c7999e6665..8bf66fe9579 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/BaseRemoteQueryLogic.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/BaseRemoteQueryLogic.java @@ -130,7 +130,7 @@ public QueryLogicTransformer getTransformer(Query settings) { return transformerInstance; } - public abstract QueryLogicTransformer createTransformer(Query settings, MarkingFunctions markingFunctions, + public abstract QueryLogicTransformer createTransformer(Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory); @Override diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/RemoteEdgeQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/RemoteEdgeQueryLogic.java index 57df85a3b6a..7dfcad41b57 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/RemoteEdgeQueryLogic.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/RemoteEdgeQueryLogic.java @@ -58,7 +58,7 @@ public void setupQuery(GenericQueryConfiguration genericConfig) throws Exception } @Override - public QueryLogicTransformer createTransformer(Query settings, MarkingFunctions markingFunctions, + public QueryLogicTransformer createTransformer(Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { return new EdgeBaseTransformer(settings, markingFunctions, responseObjectFactory, edgeFields); } @@ -134,7 +134,7 @@ public EdgeModelFields getEdgeFields() { private class EdgeBaseTransformer extends EdgeQueryTransformerSupport { - public EdgeBaseTransformer(Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, EdgeModelFields fields) { + public EdgeBaseTransformer(Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, EdgeModelFields fields) { super(settings, markingFunctions, responseObjectFactory, fields); } diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/RemoteEventQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/RemoteEventQueryLogic.java index 3d466664fa7..f692c8dd104 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/RemoteEventQueryLogic.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/RemoteEventQueryLogic.java @@ -56,7 +56,7 @@ public void setupQuery(GenericQueryConfiguration genericConfig) throws Exception } @Override - public QueryLogicTransformer createTransformer(Query settings, MarkingFunctions markingFunctions, + public QueryLogicTransformer createTransformer(Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { return new EventBaseTransformer(settings, markingFunctions, responseObjectFactory); } @@ -124,11 +124,11 @@ public EventBase next() { private class EventBaseTransformer extends EventQueryTransformerSupport { - public EventBaseTransformer(Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { + public EventBaseTransformer(Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { super("notable", settings, markingFunctions, responseObjectFactory); } - public EventBaseTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, + public EventBaseTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { super(logic, settings, markingFunctions, responseObjectFactory); } diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java index da5c4377f1a..f7b070992a3 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java @@ -719,7 +719,7 @@ public QueryLogicTransformer getTransformer(Query settings) { return this.transformerInstance; } - MarkingFunctions markingFunctions = this.getMarkingFunctions(); + MarkingFunctions markingFunctions = this.getMarkingFunctions(); ResponseObjectFactory responseObjectFactory = this.getResponseObjectFactory(); boolean reducedInSettings = false; @@ -747,7 +747,7 @@ public QueryLogicTransformer getTransformer(Query settings) { return this.transformerInstance; } - protected DocumentTransformer createDocumentTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, + protected DocumentTransformer createDocumentTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, Boolean reducedResponse) { return new DocumentTransformer(logic, settings, markingFunctions, responseObjectFactory, reducedResponse); } diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountAggregatingIterator.java b/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountAggregatingIterator.java index 08ec99cf1a7..110637c8f92 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountAggregatingIterator.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountAggregatingIterator.java @@ -57,7 +57,7 @@ public class CountAggregatingIterator extends TransformIterator { * @param markingFunctions * the marking functions */ - public CountAggregatingIterator(Iterator> iterator, Transformer transformer, MarkingFunctions markingFunctions) { + public CountAggregatingIterator(Iterator> iterator, Transformer transformer, MarkingFunctions markingFunctions) { this(iterator, transformer, markingFunctions, DEFAULT_PAGE_WAIT_TIME_MILLIS); } @@ -74,7 +74,8 @@ public CountAggregatingIterator(Iterator> iterator, Transformer * the time to wait for the next page */ @SuppressWarnings("unchecked") - public CountAggregatingIterator(Iterator> iterator, Transformer transformer, MarkingFunctions markingFunctions, long pageWaitTimeMillis) { + public CountAggregatingIterator(Iterator> iterator, Transformer transformer, MarkingFunctions markingFunctions, + long pageWaitTimeMillis) { super(iterator, transformer); this.aggregator = new CountEntryAggregator(transformer, markingFunctions); this.pageWaitTimeMillis = pageWaitTimeMillis; @@ -121,12 +122,12 @@ private Object getIntermediateEvent() { private static class CountEntryAggregator { private final AtomicLong count = new AtomicLong(0L); - private final Set cvs = new HashSet<>(); + private final Set columnVisibilities = new HashSet<>(); private final Transformer transformer; - private final MarkingFunctions markingFunctions; + private MarkingFunctions markingFunctions; - public CountEntryAggregator(Transformer transformer, MarkingFunctions markingFunctions) { + public CountEntryAggregator(Transformer transformer, MarkingFunctions markingFunctions) { this.transformer = transformer; this.markingFunctions = markingFunctions; } @@ -136,13 +137,13 @@ public void addCount(long count) { } public void addColumnVisibility(ColumnVisibility cv) { - this.cvs.add(cv); + this.columnVisibilities.add(cv); } @SuppressWarnings("unchecked") public Object getAggregatedEvent() { - if (cvs.isEmpty() && count.get() == 0L) { - cvs.add(new ColumnVisibility("")); + if (columnVisibilities.isEmpty() && count.get() == 0L) { + columnVisibilities.add(new ColumnVisibility("")); } ColumnVisibility cv = getCombinedColumnVisibility(); @@ -151,7 +152,10 @@ public Object getAggregatedEvent() { private ColumnVisibility getCombinedColumnVisibility() { try { - return markingFunctions.combine(cvs); + if (null == markingFunctions) { + markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); + } + return markingFunctions.combineVisibilities(columnVisibilities); } catch (Exception e) { log.error("Could not combine columnVisibilities for the count", e); return null; diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountResultPostprocessor.java b/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountResultPostprocessor.java index 5edb7a85dd5..43c07aaad39 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountResultPostprocessor.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/shard/CountResultPostprocessor.java @@ -19,9 +19,9 @@ public class CountResultPostprocessor implements ResultPostprocessor { private static final Logger log = Logger.getLogger(CountResultPostprocessor.class); - private final MarkingFunctions markingFunctions; + private MarkingFunctions markingFunctions; - public CountResultPostprocessor(MarkingFunctions markingFunctions) { + public CountResultPostprocessor(MarkingFunctions markingFunctions) { this.markingFunctions = markingFunctions; } @@ -70,7 +70,10 @@ public void apply(List results) { if (success) { ColumnVisibility columnVisibility = null; try { - columnVisibility = markingFunctions.combine(columnVisibilities); + if (null == markingFunctions) { + markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); + } + columnVisibility = markingFunctions.combineVisibilities(columnVisibilities); } catch (Exception e) { log.error("Could not create combined columnVisibilities for the count", e); } diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java index 261c988c016..91623cf9b74 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java @@ -476,13 +476,13 @@ public Map buildSummary(Iterator iter, long maxValues) { public static class Tuple { - private final MarkingFunctions tupleMarkingFunctions; + private MarkingFunctions markingFunctions; private long count = 0L; private long maxTimestamp = 0L; Set uniqueVisibilities = new HashSet<>(); - public Tuple(MarkingFunctions mf) { - tupleMarkingFunctions = mf; + public Tuple(MarkingFunctions markingFunctions) { + this.markingFunctions = markingFunctions; } public void aggregate(Key key, Value val) { @@ -502,17 +502,20 @@ public long getMaxTimestamp() { } public ColumnVisibility getColumnVisibility() { + Set columnVisibilities = new HashSet<>(); + for (Text t : this.uniqueVisibilities) { + columnVisibilities.add(new ColumnVisibility(t)); + } try { - Set columnVisibilities = new HashSet<>(); - for (Text t : this.uniqueVisibilities) { - columnVisibilities.add(new ColumnVisibility(t)); + if (null == markingFunctions) { + markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); } - return tupleMarkingFunctions.combine(columnVisibilities); - - } catch (MarkingFunctions.Exception e) { - logger.error("Could not create combined column visibility for the count", e); + return markingFunctions.combineVisibilities(columnVisibilities); + } catch (Exception e) { + log.warn("Invalid columnvisibility after combining!", e); return null; } + } } diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/ssdeep/SSDeepDiscoveryQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/ssdeep/SSDeepDiscoveryQueryLogic.java index 02a4d3c2dc7..f27ad4732ef 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/ssdeep/SSDeepDiscoveryQueryLogic.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/ssdeep/SSDeepDiscoveryQueryLogic.java @@ -209,7 +209,7 @@ public void setResponseObjectFactory(ResponseObjectFactory responseObjectFactory discoveryDelegate.setResponseObjectFactory(responseObjectFactory); } - public void setMarkingFunctions(MarkingFunctions markingFunctions) { + public void setMarkingFunctions(MarkingFunctions markingFunctions) { discoveryDelegate.setMarkingFunctions(markingFunctions); } diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/ssdeep/SSDeepSimilarityQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/tables/ssdeep/SSDeepSimilarityQueryTransformer.java index ccc22fd1fd5..9fbb4107c3f 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/ssdeep/SSDeepSimilarityQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/ssdeep/SSDeepSimilarityQueryTransformer.java @@ -25,7 +25,7 @@ public class SSDeepSimilarityQueryTransformer extends BaseQueryLogicTransformer< protected final ResponseObjectFactory responseObjectFactory; - public SSDeepSimilarityQueryTransformer(Query query, SSDeepSimilarityQueryConfiguration config, MarkingFunctions markingFunctions, + public SSDeepSimilarityQueryTransformer(Query query, SSDeepSimilarityQueryConfiguration config, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { super(markingFunctions); this.auths = new Authorizations(query.getQueryAuthorizations().split(",")); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/ContentQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/ContentQueryTransformer.java index 0f12a34c2d4..ac885f55d71 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/ContentQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/ContentQueryTransformer.java @@ -13,7 +13,6 @@ import datawave.core.query.logic.BaseQueryLogicTransformer; import datawave.marking.MarkingFunctions; -import datawave.marking.MarkingFunctions.Exception; import datawave.microservice.query.Query; import datawave.query.table.parser.ContentKeyValueFactory; import datawave.query.table.parser.ContentKeyValueFactory.ContentKeyValue; @@ -34,11 +33,11 @@ public class ContentQueryTransformer extends BaseQueryLogicTransformer metadataIdMap; protected final boolean decodeView; - public ContentQueryTransformer(Query query, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { + public ContentQueryTransformer(Query query, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { this(query, markingFunctions, responseObjectFactory, false); } - public ContentQueryTransformer(Query query, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, boolean decodeView) { + public ContentQueryTransformer(Query query, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, boolean decodeView) { super(markingFunctions); this.auths = new Authorizations(query.getQueryAuthorizations().split(",")); this.responseObjectFactory = responseObjectFactory; @@ -124,7 +123,7 @@ public EventBase transform(Entry entry) { ContentKeyValue ckv; try { ckv = ContentKeyValueFactory.parse(entry.getKey(), entry.getValue(), auths, markingFunctions); - } catch (Exception e1) { + } catch (MarkingFunctions.Exception e1) { throw new IllegalArgumentException("Unable to parse visibility", e1); } diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransform.java b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransform.java index ea9d6fca930..f519573ce4f 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransform.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransform.java @@ -14,7 +14,7 @@ public interface DocumentTransform extends Function,Map.Entry> { // called when adding the document transform - void initialize(Query settings, MarkingFunctions markingFunctions); + void initialize(Query settings, MarkingFunctions markingFunctions); // called after the last document is passed through to get any remaining aggregated results. Map.Entry flush(); @@ -30,11 +30,11 @@ public interface DocumentTransform extends Function,Map. class DefaultDocumentTransform implements DocumentTransform { protected Query settings; - protected MarkingFunctions markingFunctions; + protected MarkingFunctions markingFunctions; protected long queryExecutionForPageStartTime; @Override - public void initialize(Query settings, MarkingFunctions markingFunctions) { + public void initialize(Query settings, MarkingFunctions markingFunctions) { this.settings = settings; this.markingFunctions = markingFunctions; this.queryExecutionForPageStartTime = System.currentTimeMillis(); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java index 6d85f41df66..d4f2c17f377 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformer.java @@ -2,7 +2,6 @@ import java.util.ArrayList; import java.util.Collection; -import java.util.Map; import java.util.Map.Entry; import org.apache.accumulo.core.data.Key; @@ -18,6 +17,7 @@ import datawave.core.query.logic.WritesQueryMetrics; import datawave.core.query.logic.WritesResultCardinalities; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.microservice.query.Query; import datawave.query.attributes.Document; import datawave.webservice.query.result.event.EventBase; @@ -35,7 +35,7 @@ public class DocumentTransformer extends DocumentTransformerSupport,EventBase> implements WritesQueryMetrics, WritesResultCardinalities, Flushable { - private static final Logger log = Logger.getLogger(DocumentTransformerSupport.class); + private static final Logger log = Logger.getLogger(DocumentTransformer.class); /** * By default, assume each cell still has the visibility attached to it @@ -49,17 +49,17 @@ public class DocumentTransformer extends DocumentTransformerSupport> logic, Query settings, MarkingFunctions markingFunctions, + public DocumentTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { super(logic, settings, markingFunctions, responseObjectFactory); } - public DocumentTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, + public DocumentTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, Boolean reducedResponse) { super(logic, settings, markingFunctions, responseObjectFactory, reducedResponse); } - public DocumentTransformer(String tableName, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, + public DocumentTransformer(String tableName, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, Boolean reducedResponse) { super(tableName, settings, markingFunctions, responseObjectFactory, reducedResponse); } @@ -162,13 +162,13 @@ private EventBase _transform(Entry documentEntry) throws EmptyObje return output; } - protected EventBase buildResponse(Document document, Key documentKey, ColumnVisibility eventCV, String colf, String row, MarkingFunctions mf) - throws MarkingFunctions.Exception { + protected EventBase buildResponse(Document document, Key documentKey, ColumnVisibility eventCV, String colf, String row, + MarkingFunctions markingFunctions) throws MarkingFunctions.Exception { - Map markings = mf.translateFromColumnVisibility(eventCV); + Markings markings = markingFunctions.translateFromColumnVisibility(eventCV); EventBase event = null; - final Collection> documentFields = buildDocumentFields(documentKey, null, document, eventCV, mf); + final Collection> documentFields = buildDocumentFields(documentKey, null, document, eventCV, markingFunctions); // if documentFields is empty, then the response contained only timing metadata if (!documentFields.isEmpty()) { event = this.responseObjectFactory.getEvent(); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformerSupport.java b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformerSupport.java index 35625d0d6dc..4b10d82d684 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformerSupport.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformerSupport.java @@ -29,6 +29,7 @@ import datawave.core.query.logic.WritesQueryMetrics; import datawave.core.query.logic.WritesResultCardinalities; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.microservice.query.Query; import datawave.microservice.query.QueryImpl.Parameter; import datawave.microservice.querymetric.BaseQueryMetric; @@ -68,7 +69,6 @@ public abstract class DocumentTransformerSupport extends EventQueryTransfor protected Boolean reducedResponse; private static final Logger log = Logger.getLogger(DocumentTransformerSupport.class); - private static final Map EMPTY_MARKINGS = new HashMap<>(); private long sourceCount = 0; private long nextCount = 0; @@ -110,19 +110,19 @@ public abstract class DocumentTransformerSupport extends EventQueryTransfor * @param responseObjectFactory * the response object factory */ - public DocumentTransformerSupport(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, + public DocumentTransformerSupport(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { this(logic, settings, markingFunctions, responseObjectFactory, false); } - public DocumentTransformerSupport(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, + public DocumentTransformerSupport(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, Boolean reducedResponse) { this(null != logic ? logic.getTableName() : null, settings, markingFunctions, responseObjectFactory, reducedResponse); this.logic = logic; } - public DocumentTransformerSupport(String tableName, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, + public DocumentTransformerSupport(String tableName, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, Boolean reducedResponse) { super(tableName, settings, markingFunctions, responseObjectFactory); @@ -185,7 +185,7 @@ protected static Key correctKey(Key origKey) { * @return a collection of the document fields */ protected Collection> buildDocumentFields(Key documentKey, String documentName, Document document, ColumnVisibility topLevelColumnVisibility, - MarkingFunctions markingFunctions) { + MarkingFunctions markingFunctions) { // Whether the fields were added to projectFields or removed from disallowlistedFields, they user does not want them returned // If neither a projection nor a disallowlist was used then the suppressFields set should remain empty @@ -449,7 +449,7 @@ public void writeResultCardinalities() { * @return a collection of fields */ protected Collection> buildDocumentFields(Key documentKey, String fieldName, Attribute attr, ColumnVisibility topLevelColumnVisibility, - MarkingFunctions markingFunctions) { + MarkingFunctions markingFunctions) { Set> myFields = new HashSet<>(); @@ -463,16 +463,15 @@ protected Collection> buildDocumentFields(Key documentKey, String f // Use the markings on the Field if we're returning the markings to the client if (!this.reducedResponse) { try { - Map markings = markingFunctions.translateFromColumnVisibility(attr.getColumnVisibility()); + Markings markings = markingFunctions.translateFromColumnVisibility(attr.getColumnVisibility()); FieldBase field = this.makeField(fieldName, markings, attr.getColumnVisibility(), attr.getTimestamp(), attr.getData()); - MarkingFunctions.Util.populate(field, markings); myFields.add(field); } catch (Exception ex) { log.error("unable to process markings:" + ex); } } else { // noinspection RedundantCast - FieldBase f = createField(fieldName, (Long) null, attr, EMPTY_MARKINGS, (String) null); + FieldBase f = createField(fieldName, (Long) null, attr, null, (String) null); myFields.add(f); } } @@ -495,8 +494,7 @@ protected Collection> buildDocumentFields(Key documentKey, String f * mapping of markings * @return a field */ - protected FieldBase createField(final String fieldName, final long ts, final Attribute attribute, Map markings, - String columnVisibility) { + protected FieldBase createField(final String fieldName, final long ts, final Attribute attribute, Markings markings, String columnVisibility) { if (markings == null || markings.isEmpty()) { log.warn("Null or empty markings for " + fieldName + ":" + attribute); } @@ -504,16 +502,14 @@ protected FieldBase createField(final String fieldName, final long ts, final return createField(fieldName, (Long) ts, attribute, markings, columnVisibility); } - protected FieldBase createField(final String fieldName, final Long ts, final Attribute attribute, Map markings, - String columnVisibility) { + protected FieldBase createField(final String fieldName, final Long ts, final Attribute attribute, Markings markings, String columnVisibility) { if (this.transformValuePrefixFields.contains(fieldName)) { return convertHitTermField(fieldName, ts, attribute, markings, columnVisibility); } return this.makeField(fieldName, markings, columnVisibility, ts, attribute.getData()); } - private FieldBase convertHitTermField(final String fieldName, final Long ts, Attribute attribute, Map markings, - String columnVisibility) { + private FieldBase convertHitTermField(final String fieldName, final Long ts, Attribute attribute, Markings markings, String columnVisibility) { return this.makeField(fieldName, markings, columnVisibility, ts, convertMappedAttribute(attribute).getData()); } diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformer.java index ee584c5733d..8a1d6e36686 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformer.java @@ -2,7 +2,6 @@ import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.Map.Entry; import org.apache.accumulo.core.data.Key; @@ -18,6 +17,7 @@ import datawave.edge.util.EdgeValue; import datawave.edge.util.EdgeValueHelper; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.microservice.query.Query; import datawave.util.time.DateHelper; import datawave.webservice.query.result.edge.EdgeBase; @@ -26,7 +26,7 @@ public class EdgeQueryTransformer extends EdgeQueryTransformerSupport,EdgeBase> implements CacheableLogic { private Logger log = Logger.getLogger(EdgeQueryTransformer.class); - public EdgeQueryTransformer(Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, EdgeModelFields fields) { + public EdgeQueryTransformer(Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, EdgeModelFields fields) { super(settings, markingFunctions, responseObjectFactory, fields); } @@ -36,11 +36,11 @@ public EdgeBase transform(Entry entry) { EdgeKey edgeKey = EdgeKey.decode(entry.getKey()); Value value = entry.getValue(); - EdgeBase edge = (EdgeBase) this.responseObjectFactory.getEdge(); + EdgeBase edge = this.responseObjectFactory.getEdge(); boolean statsEdge = edgeKey.isStatsKey(); try { - Map markings = markingFunctions.translateFromColumnVisibilityForAuths(new ColumnVisibility(edgeKey.getColvis()), auths); + Markings markings = markingFunctions.translateFromColumnVisibilityForAuths(new ColumnVisibility(edgeKey.getColvis()), auths); edge.setMarkings(markings); edge.setEdgeType(edgeKey.getType()); edge.setEdgeRelationship(edgeKey.getRelationship()); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformerSupport.java b/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformerSupport.java index 5be63b09779..c637e9b6407 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformerSupport.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/EdgeQueryTransformerSupport.java @@ -2,17 +2,16 @@ import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.Authorizations; -import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.builder.HashCodeBuilder; -import com.google.common.collect.Sets; import com.google.protobuf.InvalidProtocolBufferException; import datawave.core.query.cachedresults.CacheableLogic; @@ -20,6 +19,7 @@ import datawave.edge.model.EdgeModelFields; import datawave.edge.util.EdgeValue; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.microservice.query.Query; import datawave.webservice.query.cachedresults.CacheableQueryRow; import datawave.webservice.query.exception.QueryException; @@ -33,7 +33,8 @@ public abstract class EdgeQueryTransformerSupport extends BaseQueryLogicTra protected ResponseObjectFactory responseObjectFactory; protected EdgeModelFields fields; - public EdgeQueryTransformerSupport(Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, EdgeModelFields fields) { + public EdgeQueryTransformerSupport(Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, + EdgeModelFields fields) { super(markingFunctions); this.responseObjectFactory = responseObjectFactory; auths = new Authorizations(settings.getQueryAuthorizations().split(",")); @@ -99,16 +100,14 @@ public BaseQueryResponse createResponse(List resultList) { try { EdgeQueryResponseBase response = responseObjectFactory.getEdgeQueryResponse(); - Set uniqueColumnVisibilities = Sets.newHashSet(); + Set> uniqueMarkings = new HashSet<>(); for (Object result : resultList) { EdgeBase edge = (EdgeBase) result; - Map markings = edge.getMarkings(); - uniqueColumnVisibilities.add(this.markingFunctions.translateToColumnVisibility(markings)); + uniqueMarkings.add(edge.getMarkings()); response.addEdge(edge); } - ColumnVisibility combinedVisibility = this.markingFunctions.combine(uniqueColumnVisibilities); - response.setMarkings(this.markingFunctions.translateFromColumnVisibility(combinedVisibility)); + response.setMarkings(markingFunctions.combine(uniqueMarkings)); return response; } catch (Exception ex) { throw new RuntimeException("could not handle markings in resultList ", ex); @@ -171,7 +170,7 @@ public CacheableQueryRow writeToCache(Object o) throws QueryException { @Override public Object readFromCache(CacheableQueryRow cacheableQueryRow) { - Map markings = cacheableQueryRow.getMarkings(); + Markings markings = cacheableQueryRow.getMarkings(); EdgeBase edge = (EdgeBase) responseObjectFactory.getEdge(); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryDataDecorator.java b/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryDataDecorator.java index 640dc36b34c..f6155c3ef77 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryDataDecorator.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryDataDecorator.java @@ -9,6 +9,7 @@ import com.google.common.collect.Multimap; +import datawave.marking.Markings; import datawave.webservice.query.result.event.FieldBase; import datawave.webservice.query.result.event.ResponseObjectFactory; @@ -33,7 +34,7 @@ public void decorateData(Multimap data) { // multiple value source fields for the substitution value not supported -- use the first one Iterator collectionSourceItr = collectionSourceOfData.iterator(); FieldBase sourceOfData = collectionSourceItr.next(); - Map markings = sourceOfData.getMarkings(); + Markings markings = sourceOfData.getMarkings(); String id = sourceOfData.getValueString(); String newValue = entry.getValue().replace("@field_value@", id); @@ -61,7 +62,7 @@ public void decorateData(Multimap data) { } } - private FieldBase makeField(String name, Map markings, String columnVisibility, Long timestamp, Object value) { + private FieldBase makeField(String name, Markings markings, String columnVisibility, Long timestamp, Object value) { FieldBase field = this.responseObjectFactory.getField(); field.setName(name); field.setMarkings(markings); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java index 3b6eef77141..c4469a7452d 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformer.java @@ -5,7 +5,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; -import java.util.Map; import java.util.Map.Entry; import java.util.Set; @@ -18,7 +17,7 @@ import datawave.core.query.cachedresults.CacheableLogic; import datawave.core.query.logic.BaseQueryLogic; import datawave.marking.MarkingFunctions; -import datawave.marking.MarkingFunctions.Exception; +import datawave.marking.Markings; import datawave.microservice.query.Query; import datawave.query.Constants; import datawave.query.parser.EventFields; @@ -30,11 +29,13 @@ public class EventQueryTransformer extends EventQueryTransformerSupport,EventBase> implements CacheableLogic { - public EventQueryTransformer(String tableName, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { + private MarkingFunctions markingFunctions; + + public EventQueryTransformer(String tableName, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { super(tableName, settings, markingFunctions, responseObjectFactory); } - public EventQueryTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, + public EventQueryTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { super(logic, settings, markingFunctions, responseObjectFactory); } @@ -52,13 +53,18 @@ public EventBase transform(Entry entry) { } EventBase event = this.responseObjectFactory.getEvent(); - Map markings = null; + if (null == markingFunctions) { + markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); + } + + Markings markings; try { markings = this.markingFunctions.translateFromColumnVisibilityForAuths(new ColumnVisibility(key.getColumnVisibility()), this.auths); } catch (Exception e) { log.error("could not translate " + key.getColumnVisibility() + " to markings, skipping entry"); return null; } + if (null == markings || markings.isEmpty()) { // can't process this one because we did not have valid security markings log.error("Transformer visibility interpreter was null, skipping entry"); @@ -86,8 +92,8 @@ public EventBase transform(Entry entry) { String origFieldName = null; String fieldName = null; - // Hold unique Column Visibilities and merge them at the end - // for the overall event ColumnVisibility. + // Hold unique visibilities and merge them at the end + // for the overall event visibility. Set visibilitiesToMerge = new HashSet<>(); for (Entry> e : eventFields.asMap().entrySet()) { @@ -101,14 +107,15 @@ public EventBase transform(Entry entry) { for (FieldValue fv : e.getValue()) { visibilitiesToMerge.add(fv.getVisibility()); try { - Map fieldMarkings = this.markingFunctions.translateFromColumnVisibility(fv.getVisibility()); + Markings fieldMarkings = this.markingFunctions.translateFromColumnVisibility(fv.getVisibility()); String value = new String(fv.getValue(), StandardCharsets.UTF_8); // if this is a content field name, then replace the value with the uid if (getContentFieldNames().contains(fieldName)) { value = baseUid; } - values.add(this.makeField(fieldName, fieldMarkings, new String(fv.getVisibility().getExpression()), entry.getKey().getTimestamp(), value)); - } catch (Exception e1) { + values.add(this.makeField(fieldName, fieldMarkings, fieldMarkings.toAccessExpression().getExpression(), entry.getKey().getTimestamp(), + value)); + } catch (MarkingFunctions.Exception e1) { throw new RuntimeException("could not make markings from: " + fv.getVisibility()); } } @@ -116,11 +123,12 @@ public EventBase transform(Entry entry) { ColumnVisibility columnVisibility = null; try { - columnVisibility = this.markingFunctions.combine(visibilitiesToMerge); - event.setMarkings(this.markingFunctions.translateFromColumnVisibility(columnVisibility)); + columnVisibility = markingFunctions.combineVisibilities(visibilitiesToMerge); + event.setMarkings(markingFunctions.translateFromColumnVisibility(columnVisibility)); } catch (Exception e1) { throw new RuntimeException("could not make markings from: " + columnVisibility); } + event.setFields(new ArrayList<>(values)); Metadata metadata = new Metadata(); @@ -145,7 +153,7 @@ public EventBase transform(Entry entry) { // assign an estimate of the event size // in practice this is about 6 times the size of the kryo bytes - event.setSizeInBytes(entry.getValue().getSize() * 6); + event.setSizeInBytes(entry.getValue().getSize() * 6L); return event; } diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformerSupport.java b/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformerSupport.java index edf372e7021..353ce22d8bc 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformerSupport.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/EventQueryTransformerSupport.java @@ -21,7 +21,9 @@ import datawave.core.query.cachedresults.CacheableLogic; import datawave.core.query.logic.BaseQueryLogic; import datawave.core.query.logic.BaseQueryLogicTransformer; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.microservice.query.Query; import datawave.microservice.query.QueryImpl.Parameter; import datawave.query.model.QueryModel; @@ -57,7 +59,7 @@ public abstract class EventQueryTransformerSupport extends BaseQueryLogicTr protected String tableName; protected ResponseObjectFactory responseObjectFactory; - public EventQueryTransformerSupport(String tableName, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { + public EventQueryTransformerSupport(String tableName, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { super(markingFunctions); this.settings = settings; this.auths = new Authorizations(settings.getQueryAuthorizations().split(",")); @@ -65,7 +67,7 @@ public EventQueryTransformerSupport(String tableName, Query settings, MarkingFun this.responseObjectFactory = responseObjectFactory; } - public EventQueryTransformerSupport(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, + public EventQueryTransformerSupport(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { this(logic.getTableName(), settings, markingFunctions, responseObjectFactory); this.logic = logic; @@ -101,7 +103,7 @@ public CacheableQueryRow writeToCache(Object o) throws QueryException { @Override public Object readFromCache(CacheableQueryRow cacheableQueryRow) { - Map markings = cacheableQueryRow.getMarkings(); + Markings markings = cacheableQueryRow.getMarkings(); String dataType = cacheableQueryRow.getDataType(); String internalId = cacheableQueryRow.getEventId(); String row = cacheableQueryRow.getRow(); @@ -121,7 +123,7 @@ public Object readFromCache(CacheableQueryRow cacheableQueryRow) { for (Entry entry : columnValueMap.entrySet()) { String columnName = entry.getKey(); String columnValue = entry.getValue(); - Map columnMarkings = cacheableQueryRow.getColumnMarkings(columnName); + Markings columnMarkings = cacheableQueryRow.getColumnMarkings(columnName); String columnVisibility = cacheableQueryRow.getColumnVisibility(columnName); Long columnTimestamp = cacheableQueryRow.getColumnTimestamp(columnName); FieldBase field = this.makeField(columnName, columnMarkings, columnVisibility, columnTimestamp, columnValue); @@ -150,7 +152,7 @@ public BaseQueryResponse createResponse(List resultList) { return response; } - protected FieldBase makeField(String name, Map markings, String columnVisibility, Long timestamp, Object value) { + protected FieldBase makeField(String name, Markings markings, String columnVisibility, Long timestamp, Object value) { FieldBase field = this.responseObjectFactory.getField(); field.setName(name); field.setMarkings(markings); @@ -160,9 +162,9 @@ protected FieldBase makeField(String name, Map markings, Strin return field; } - protected FieldBase makeField(String name, Map markings, ColumnVisibility columnVisibility, Long timestamp, Object value) { + protected FieldBase makeField(String name, Markings markings, ColumnVisibility columnVisibility, Long timestamp, Object value) { FieldBase field = makeField(name, markings, (String) null, timestamp, value); - field.setColumnVisibility(columnVisibility); + field.setColumnVisibility(AccessExpressionUtil.toAccessExpression(columnVisibility)); return field; } diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java index 6fb9c45ddf2..c5ab0b18e30 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/FacetedTransformer.java @@ -7,6 +7,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import java.util.stream.Collectors; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; @@ -19,6 +20,7 @@ import datawave.core.query.exception.EmptyObjectException; import datawave.core.query.logic.BaseQueryLogic; import datawave.data.type.StringType; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; import datawave.microservice.query.Query; import datawave.query.attributes.Attribute; @@ -36,20 +38,22 @@ public class FacetedTransformer extends DocumentTransformerSupport markingFunctions; + /* * By default, assume each cell still has the visibility attached to it */ - public FacetedTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, + public FacetedTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { super(logic, settings, markingFunctions, responseObjectFactory); } - public FacetedTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, + public FacetedTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, Boolean reducedResponse) { super(logic, settings, markingFunctions, responseObjectFactory, reducedResponse); } - public FacetedTransformer(String tableName, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, + public FacetedTransformer(String tableName, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, Boolean reducedResponse) { super(tableName, settings, markingFunctions, responseObjectFactory, reducedResponse); } @@ -70,7 +74,7 @@ public FacetedTransformer(String tableName, Query settings, MarkingFunctions mar * @return list of facets */ protected Collection buildFacets(Key documentKey, String fieldName, Document document, ColumnVisibility topLevelColumnVisibility, - MarkingFunctions markingFunctions) { + MarkingFunctions markingFunctions) { Set myFields = new HashSet<>(); @@ -104,7 +108,7 @@ protected Collection buildFacets(Key documentKey, String f * @return list of facets */ protected Collection buildFacets(Key documentKey, String fieldName, Attribute attr, ColumnVisibility topLevelColumnVisibility, - MarkingFunctions markingFunctions) { + MarkingFunctions markingFunctions) { Set myFields = new HashSet<>(); @@ -126,9 +130,8 @@ protected Collection buildFacets(Key documentKey, String f FieldCardinalityBase fc = this.responseObjectFactory.getFieldCardinality(); fc.setField(v.getFieldName()); - fc.setMarkings(markingFunctions.translateFromColumnVisibilityForAuths(attr.getColumnVisibility(), auths)); // reduces colvis based on - // visibility - fc.setColumnVisibility(new String(markingFunctions.translateToColumnVisibility(fc.getMarkings()).flatten())); + fc.setMarkings(markingFunctions.translateFromColumnVisibilityForAuths(attr.getColumnVisibility(), auths)); + fc.setColumnVisibility(AccessExpressionUtil.normalize(fc.getMarkings().toAccessExpression()).getExpression()); fc.setLower(v.getFloorValue()); fc.setUpper(v.getCeilingValue()); fc.setCardinality(v.getEstimate().cardinality()); @@ -145,14 +148,14 @@ protected Collection buildFacets(Key documentKey, String f return myFields; } - protected FacetsBase buildResponse(Document document, Key documentKey, ColumnVisibility eventCV, String colf, String row, MarkingFunctions mf) - throws MarkingFunctions.Exception { + protected FacetsBase buildResponse(Document document, Key documentKey, ColumnVisibility eventCV, String colf, String row, + MarkingFunctions markingFunctions) throws MarkingFunctions.Exception { FacetsBase facetedResponse = responseObjectFactory.getFacets(); - final Collection documentFields = buildFacets(documentKey, null, document, eventCV, mf); + final Collection documentFields = buildFacets(documentKey, null, document, eventCV, markingFunctions); - facetedResponse.setMarkings(mf.translateFromColumnVisibility(eventCV)); + facetedResponse.setMarkings(markingFunctions.translateFromColumnVisibility(eventCV)); facetedResponse.setFields(new ArrayList<>(documentFields)); // assign an estimate of the event size based on the document size @@ -166,7 +169,7 @@ protected FacetsBase buildResponse(Document document, Key documentKey, ColumnVis @Override public BaseQueryResponse createResponse(List resultList) { FacetQueryResponseBase response = responseObjectFactory.getFacetQueryResponse(); - Set combinedColumnVisibility = new HashSet<>(); + Collection visibilities = new HashSet<>(); for (Object result : resultList) { FacetsBase facet = (FacetsBase) result; @@ -177,15 +180,25 @@ public BaseQueryResponse createResponse(List resultList) { for (FieldCardinalityBase fcb : facet.getFields()) { if (StringUtils.isNotBlank(fcb.getColumnVisibility())) { - combinedColumnVisibility.add(new ColumnVisibility(fcb.getColumnVisibility())); + visibilities.add(fcb.getColumnVisibility()); } } } + ColumnVisibility cv; try { - ColumnVisibility columnVisibility = this.markingFunctions.combine(combinedColumnVisibility); - response.setMarkings(this.markingFunctions.translateFromColumnVisibility(columnVisibility)); + if (null == markingFunctions) { + markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); + } + // Calculate the columnVisibility for this key from the combiner. + cv = markingFunctions.combineVisibilities(visibilities.stream().map(ColumnVisibility::new).collect(Collectors.toList())); + } catch (Exception e) { + log.error("Could not create combined columnVisibility for the count", e); + return null; + } + try { + response.setMarkings(markingFunctions.translateFromColumnVisibility(cv)); } catch (MarkingFunctions.Exception e) { log.warn(e); // original ignored these exceptions diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/FieldIndexCountQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/FieldIndexCountQueryTransformer.java index f570fa4fc15..28ee4430bf8 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/FieldIndexCountQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/FieldIndexCountQueryTransformer.java @@ -16,6 +16,7 @@ import datawave.core.query.logic.BaseQueryLogicTransformer; import datawave.data.hash.UID; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.microservice.query.Query; import datawave.query.Constants; import datawave.query.tables.shard.FieldIndexCountQueryLogic.Tuple; @@ -37,7 +38,7 @@ public class FieldIndexCountQueryTransformer extends BaseQueryLogicTransformer> logic = null; private ResponseObjectFactory responseObjectFactory; - public FieldIndexCountQueryTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, + public FieldIndexCountQueryTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { super(markingFunctions); this.auths = new Authorizations(settings.getQueryAuthorizations().split(",")); @@ -59,10 +60,10 @@ public EventBase transform(Entry entry) { String key = entry.getKey(); Tuple val = entry.getValue(); - Map markings = null; + Markings markings = null; try { - markings = this.markingFunctions.translateFromColumnVisibilityForAuths(val.getColumnVisibility(), this.auths); - } catch (Exception e) { + markings = markingFunctions.translateFromColumnVisibilityForAuths(val.getColumnVisibility(), this.auths); + } catch (MarkingFunctions.Exception e) { log.error("could not translate " + val.getColumnVisibility() + " to markings, skipping entry"); return null; } @@ -145,7 +146,7 @@ public BaseQueryResponse createResponse(List resultList) { response.setFields(variableFieldList); response.setEvents(eventList); response.setReturnedEvents((long) eventList.size()); - return (BaseQueryResponse) response; + return response; } @Override @@ -171,7 +172,7 @@ public CacheableQueryRow writeToCache(Object o) throws QueryException { return cqo; } - private FieldBase makeField(String name, Map markings, String columnVisibility, Long timestamp, Object value) { + private FieldBase makeField(String name, Markings markings, String columnVisibility, Long timestamp, Object value) { FieldBase field = this.responseObjectFactory.getField(); field.setName(name); field.setMarkings(markings); @@ -186,7 +187,7 @@ public Object readFromCache(CacheableQueryRow cacheableQueryRow) { if (this.variableFieldList == null) { this.variableFieldList = cacheableQueryRow.getVariableColumnNames(); } - Map markings = cacheableQueryRow.getMarkings(); + Markings markings = cacheableQueryRow.getMarkings(); String dataType = cacheableQueryRow.getDataType(); String internalId = cacheableQueryRow.getEventId(); String row = cacheableQueryRow.getRow(); @@ -206,7 +207,7 @@ public Object readFromCache(CacheableQueryRow cacheableQueryRow) { for (Map.Entry entry : columnValueMap.entrySet()) { String columnName = entry.getKey(); String columnValue = entry.getValue(); - Map columnMarkings = cacheableQueryRow.getColumnMarkings(columnName); + Markings columnMarkings = cacheableQueryRow.getColumnMarkings(columnName); String columnVisibility = cacheableQueryRow.getColumnVisibility(columnName); Long columnTimestamp = cacheableQueryRow.getColumnTimestamp(columnName); FieldBase field = this.makeField(columnName, columnMarkings, columnVisibility, columnTimestamp, columnValue); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/GroupingDocumentTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/GroupingDocumentTransformer.java index 9c764087cdf..eb798ad5943 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/GroupingDocumentTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/GroupingDocumentTransformer.java @@ -22,7 +22,9 @@ import com.google.common.collect.TreeMultimap; import datawave.core.query.logic.BaseQueryLogic; +import datawave.marking.AccessExpressionMarkings; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.microservice.query.Query; import datawave.query.model.QueryModel; import datawave.query.tables.ShardQueryLogic; @@ -39,19 +41,19 @@ public class GroupingDocumentTransformer extends DocumentTransformer { private List groupFieldsList; private Map> fieldMap = Maps.newHashMap(); - public GroupingDocumentTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, + public GroupingDocumentTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, Collection groupFieldsSet) { super(logic, settings, markingFunctions, responseObjectFactory); createGroupFieldsList(groupFieldsSet); } - public GroupingDocumentTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, + public GroupingDocumentTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, Collection groupFieldsSet, Boolean reducedResponse) { super(logic, settings, markingFunctions, responseObjectFactory, reducedResponse); createGroupFieldsList(groupFieldsSet); } - public GroupingDocumentTransformer(String tableName, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, + public GroupingDocumentTransformer(String tableName, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, Collection groupFieldsSet, Boolean reducedResponse) { super(tableName, settings, markingFunctions, responseObjectFactory, reducedResponse); createGroupFieldsList(groupFieldsSet); @@ -86,7 +88,7 @@ public BaseQueryResponse createResponse(List resultList) { } protected BaseQueryResponse createGroupedResponse(Multiset>> multiset) { - Map markings = Maps.newHashMap(); + Markings markings = AccessExpressionMarkings.builder().build(); EventQueryResponseBase response = this.responseObjectFactory.getEventQueryResponse(); List events = new ArrayList<>(); for (Collection> entry : multiset.elementSet()) { @@ -128,7 +130,7 @@ private Multimap getFieldToFieldWithGroupingContextMap(Collection log.trace(this.groupFieldsList + " contains " + shorterName); FieldBase created = null; try { - created = this.makeField(shortName, this.markingFunctions.translateFromColumnVisibility(new ColumnVisibility(field.getColumnVisibility())), + created = this.makeField(shortName, markingFunctions.translateFromColumnVisibility(new ColumnVisibility(field.getColumnVisibility())), field.getColumnVisibility(), 0L, field.getValueOfTypedValue()); } catch (Exception ex) { log.error(ex); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/GroupingTransform.java b/warehouse/query-core/src/main/java/datawave/query/transformer/GroupingTransform.java index abfb8bd6ba4..d4b21fa52e5 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/GroupingTransform.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/GroupingTransform.java @@ -70,7 +70,7 @@ public class GroupingTransform extends DocumentTransform.DefaultDocumentTransfor * @param markingFunctions * the marking functions */ - public GroupingTransform(GroupFields groupFields, MarkingFunctions markingFunctions, long queryExecutionForPageTimeout) { + public GroupingTransform(GroupFields groupFields, MarkingFunctions markingFunctions, long queryExecutionForPageTimeout) { super.initialize(settings, markingFunctions); this.queryExecutionForPageTimeout = queryExecutionForPageTimeout; this.groups = new Groups(); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/ParentDocumentTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/ParentDocumentTransformer.java index 9b96d420a25..27d5779f66e 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/ParentDocumentTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/ParentDocumentTransformer.java @@ -17,12 +17,12 @@ public class ParentDocumentTransformer extends DocumentTransformer { - public ParentDocumentTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, + public ParentDocumentTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { super(logic, settings, markingFunctions, responseObjectFactory); } - public ParentDocumentTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, + public ParentDocumentTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, Boolean cellLevelVisibility) { super(logic, settings, markingFunctions, responseObjectFactory, cellLevelVisibility); } diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/ShardIndexQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/ShardIndexQueryTransformer.java index 8a1d6f8a60e..cc0869b6a65 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/ShardIndexQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/ShardIndexQueryTransformer.java @@ -19,7 +19,7 @@ import datawave.core.query.logic.BaseQueryLogicTransformer; import datawave.ingest.protobuf.Uid; import datawave.marking.MarkingFunctions; -import datawave.marking.MarkingFunctions.Exception; +import datawave.marking.Markings; import datawave.microservice.query.Query; import datawave.query.model.QueryModel; import datawave.webservice.query.cachedresults.CacheableQueryRow; @@ -40,7 +40,7 @@ public class ShardIndexQueryTransformer extends BaseQueryLogicTransformer> logic, Query settings, MarkingFunctions markingFunctions, + public ShardIndexQueryTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, QueryModel qm) { super(markingFunctions); this.responseObjectFactory = responseObjectFactory; @@ -66,10 +66,10 @@ public EventBase transform(Entry input) { EventBase event = responseObjectFactory.getEvent(); ColumnVisibility columnVisibility = new ColumnVisibility(entry.getKey().getColumnVisibility()); - Map markings; + Markings markings; try { - markings = this.markingFunctions.translateFromColumnVisibilityForAuths(columnVisibility, this.auths); - } catch (Exception e1) { + markings = markingFunctions.translateFromColumnVisibilityForAuths(columnVisibility, this.auths); + } catch (MarkingFunctions.Exception e) { throw new RuntimeException("could not make markings from: " + columnVisibility); } event.setMarkings(markings); @@ -113,7 +113,7 @@ public EventBase transform(Entry input) { return event; } - private FieldBase makeField(String name, Map markings, String columnVisibility, Long timestamp, Object value) { + private FieldBase makeField(String name, Markings markings, String columnVisibility, Long timestamp, Object value) { FieldBase field = this.responseObjectFactory.getField(); field.setName(name); field.setMarkings(markings); @@ -166,7 +166,7 @@ public Object readFromCache(CacheableQueryRow cacheableQueryRow) { if (this.variableFieldList == null) { this.variableFieldList = cacheableQueryRow.getVariableColumnNames(); } - Map markings = cacheableQueryRow.getMarkings(); + Markings markings = cacheableQueryRow.getMarkings(); String dataType = cacheableQueryRow.getDataType(); String internalId = cacheableQueryRow.getEventId(); String row = cacheableQueryRow.getRow(); @@ -189,7 +189,7 @@ public Object readFromCache(CacheableQueryRow cacheableQueryRow) { String columnValue = entry.getValue(); String columnVisibility = cacheableQueryRow.getColumnVisibility(columnName); Long columnTimestamp = cacheableQueryRow.getColumnTimestamp(columnName); - Map columnMarkings = cacheableQueryRow.getColumnMarkings(columnName); + Markings columnMarkings = cacheableQueryRow.getColumnMarkings(columnName); FieldBase field = responseObjectFactory.getField(); field.setName(columnName); field.setMarkings(columnMarkings); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/ShardQueryCountTableTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/ShardQueryCountTableTransformer.java index 95b852c455b..82c50e5714b 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/ShardQueryCountTableTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/ShardQueryCountTableTransformer.java @@ -11,8 +11,9 @@ import datawave.core.query.cachedresults.CacheableLogic; import datawave.core.query.logic.BaseQueryLogicTransformer; +import datawave.marking.AccessExpressionUtil; import datawave.marking.MarkingFunctions; -import datawave.marking.MarkingFunctions.Exception; +import datawave.marking.Markings; import datawave.microservice.query.Query; import datawave.query.Constants; import datawave.webservice.query.cachedresults.CacheableQueryRow; @@ -34,7 +35,7 @@ public class ShardQueryCountTableTransformer extends BaseQueryLogicTransformer variableFieldList = null; private ResponseObjectFactory responseObjectFactory; - public ShardQueryCountTableTransformer(Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { + public ShardQueryCountTableTransformer(Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { super(markingFunctions); this.responseObjectFactory = responseObjectFactory; this.auths = new Authorizations(settings.getQueryAuthorizations().split(",")); @@ -46,16 +47,16 @@ public EventBase transform(Entry untypedEntry) { Long count = untypedEntry.getKey(); ColumnVisibility vis = untypedEntry.getValue(); - Map markings; + EventBase e = this.responseObjectFactory.getEvent(); + + Markings markings; try { markings = markingFunctions.translateFromColumnVisibilityForAuths(vis, auths); - } catch (Exception e1) { + } catch (MarkingFunctions.Exception e1) { throw new IllegalArgumentException("Unable to translate markings", e1); } - EventBase e = this.responseObjectFactory.getEvent(); e.setMarkings(markings); - FieldBase field = this.makeField(COUNT_CELL, markings, vis, System.currentTimeMillis(), count); e.setMarkings(markings); @@ -72,11 +73,11 @@ public EventBase transform(Entry untypedEntry) { return e; } - private FieldBase makeField(String name, Map markings, ColumnVisibility columnVisibility, Long timestamp, Object value) { + private FieldBase makeField(String name, Markings markings, ColumnVisibility columnVisibility, Long timestamp, Object value) { FieldBase field = this.responseObjectFactory.getField(); field.setName(name); field.setMarkings(markings); - field.setColumnVisibility(columnVisibility); + field.setColumnVisibility(AccessExpressionUtil.toAccessExpression(columnVisibility)); field.setTimestamp(timestamp); field.setValue(value); return field; @@ -125,7 +126,7 @@ public Object readFromCache(CacheableQueryRow cacheableQueryRow) { if (this.variableFieldList == null) { this.variableFieldList = cacheableQueryRow.getVariableColumnNames(); } - Map markings = cacheableQueryRow.getMarkings(); + Markings markings = cacheableQueryRow.getMarkings(); String dataType = cacheableQueryRow.getDataType(); String internalId = cacheableQueryRow.getEventId(); String row = cacheableQueryRow.getRow(); @@ -147,7 +148,7 @@ public Object readFromCache(CacheableQueryRow cacheableQueryRow) { String columnValue = entry.getValue(); String columnVisibility = cacheableQueryRow.getColumnVisibility(columnName); Long columnTimestamp = cacheableQueryRow.getColumnTimestamp(columnName); - Map columnMarkings = cacheableQueryRow.getColumnMarkings(columnName); + Markings columnMarkings = cacheableQueryRow.getColumnMarkings(columnName); FieldBase field = responseObjectFactory.getField(); field.setName(columnName); field.setMarkings(columnMarkings); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/TagCloudTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/TagCloudTransformer.java index a1b3431a7a5..5dadd57ddfa 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/TagCloudTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/TagCloudTransformer.java @@ -6,9 +6,13 @@ import java.util.Set; import java.util.stream.Collectors; +import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.core.security.ColumnVisibility; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import datawave.core.query.logic.BaseQueryLogicTransformer; import datawave.marking.MarkingFunctions; @@ -35,12 +39,14 @@ public class TagCloudTransformer extends BaseQueryLogicTransformer> transformers; + private static final Logger log = LoggerFactory.getLogger(TagCloudTransformer.class); + public enum ResponseVersion { V1, V2 } // TODO-crwill9 pass in state data, not the state object so this is more reusable - public TagCloudTransformer(String responseVersion, Query query, KeywordQueryState state, MarkingFunctions markingFunctions, + public TagCloudTransformer(String responseVersion, Query query, KeywordQueryState state, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, Set> transformers) { super(markingFunctions); this.responseVersion = getResponseVersion(responseVersion); @@ -183,9 +189,7 @@ private void configureV1TagCloud(TagCloudBase base, TagCloud tagCloud) { } } } - if (!tagCloud.getVisibility().isEmpty()) { - base.setMarkings(tagCloud.getVisibility()); - } + convertAndSetMarkings(base, tagCloud.getVisibility()); base.setTags(generateTagCloudEntries(tagCloud)); } @@ -193,12 +197,21 @@ private void configureV2TagCloud(TagCloudBase base, TagCloud tagCloud) { if (tagCloud.getMetadata() != null && !tagCloud.getMetadata().isEmpty()) { base.setMetadata(tagCloud.getMetadata()); } - if (!tagCloud.getVisibility().isEmpty()) { - base.setMarkings(tagCloud.getVisibility()); - } + convertAndSetMarkings(base, tagCloud.getVisibility()); base.setTags(generateTagCloudEntries(tagCloud)); } + @SuppressWarnings("unchecked") + private void convertAndSetMarkings(TagCloudBase base, AccessExpression visibility) { + if (visibility != null && !visibility.getExpression().isEmpty()) { + try { + base.setMarkings(markingFunctions.translateFromColumnVisibility(new ColumnVisibility(visibility.getExpression()))); + } catch (MarkingFunctions.Exception e) { + throw new RuntimeException(e); + } + } + } + private void configureTagCloud(TagCloudBase base, TagCloud tagCloud) { if (responseVersion == ResponseVersion.V1) { configureV1TagCloud(base, tagCloud); diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/TermFrequencyQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/TermFrequencyQueryTransformer.java index ca07a9f3be4..7002e2edab9 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/TermFrequencyQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/TermFrequencyQueryTransformer.java @@ -28,7 +28,7 @@ public class TermFrequencyQueryTransformer extends BaseQueryLogicTransformer markingFunctions, ResponseObjectFactory responseObjectFactory) { super(markingFunctions); this.query = query; this.responseObjectFactory = responseObjectFactory; diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/annotation/AnnotationHitsTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/annotation/AnnotationHitsTransformer.java index 7b2688522b9..a1788e9d3d3 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/annotation/AnnotationHitsTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/annotation/AnnotationHitsTransformer.java @@ -122,7 +122,7 @@ public AnnotationHitsTransformer(ShardQueryConfiguration shardQueryConfig, @Null } @Override - public void initialize(Query settings, MarkingFunctions markingFunctions) { + public void initialize(Query settings, MarkingFunctions markingFunctions) { super.initialize(settings, markingFunctions); // handle query parameters for configuration overrides diff --git a/warehouse/query-core/src/main/java/datawave/query/util/HasMarkingFunctions.java b/warehouse/query-core/src/main/java/datawave/query/util/HasMarkingFunctions.java index 805e5854198..a3f6e37839b 100644 --- a/warehouse/query-core/src/main/java/datawave/query/util/HasMarkingFunctions.java +++ b/warehouse/query-core/src/main/java/datawave/query/util/HasMarkingFunctions.java @@ -4,8 +4,8 @@ public interface HasMarkingFunctions { - void setMarkingFunctions(MarkingFunctions markingFunctions); + void setMarkingFunctions(MarkingFunctions markingFunctions); - MarkingFunctions getMarkingFunctions(); + MarkingFunctions getMarkingFunctions(); } diff --git a/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleEvent.java b/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleEvent.java index fa2b205ef8e..2f0c9848edb 100644 --- a/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleEvent.java +++ b/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleEvent.java @@ -5,7 +5,6 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; -import java.util.Map; import javax.xml.bind.annotation.XmlAccessOrder; import javax.xml.bind.annotation.XmlAccessType; @@ -15,8 +14,8 @@ import javax.xml.bind.annotation.XmlElementWrapper; import javax.xml.bind.annotation.XmlTransient; -import com.google.common.collect.Maps; - +import datawave.marking.AccessExpressionMarkings; +import datawave.marking.Markings; import datawave.webservice.query.data.ObjectSizeOf; import io.protostuff.Input; import io.protostuff.Message; @@ -30,7 +29,7 @@ public class SimpleEvent extends EventBase implements S private static final long serialVersionUID = -1963967131747306439L; @XmlElementWrapper(name = "Markings") - private Map markings; + private Markings markings; @XmlElement(name = "Metadata") private Metadata metadata = null; @@ -103,7 +102,7 @@ public void writeTo(Output output, SimpleEvent message) throws IOException { } } if (message.markings != null) { - output.writeObject(3, message.markings, MapSchema.SCHEMA, false); + output.writeObject(3, (AccessExpressionMarkings) message.markings, AccessExpressionMarkings.SCHEMA, false); } } @@ -121,9 +120,9 @@ public void mergeFrom(Input input, SimpleEvent message) throws IOException { message.fields.add(f); break; case 3: - if (message.markings == null) - message.markings = Maps.newHashMap(); - input.mergeObject(message.markings, MapSchema.SCHEMA); + message.markings = AccessExpressionMarkings.builder().build(); + input.mergeObject((AccessExpressionMarkings) message.markings, AccessExpressionMarkings.SCHEMA); + break; default: input.handleUnknownField(number, this); break; @@ -139,8 +138,6 @@ public String getFieldName(int number) { return "fields"; case 3: return "markings"; - case 4: - return "payload"; default: return null; } diff --git a/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleField.java b/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleField.java index 8363d8805cf..cffb337c7a1 100644 --- a/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleField.java +++ b/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleField.java @@ -17,9 +17,9 @@ import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; -import com.google.common.collect.Maps; - import datawave.data.type.Type; +import datawave.marking.AccessExpressionMarkings; +import datawave.marking.Markings; import datawave.webservice.query.util.TypedValue; import io.protostuff.Input; import io.protostuff.Message; @@ -38,7 +38,7 @@ public class SimpleField extends FieldBase implements Serializable, @XmlAttribute(name = "columnVisibility") private String columnVisibility; @XmlElementWrapper(name = "Markings") - private Map markings; + private Markings markings; @XmlAttribute(name = "timestamp") private Long timestamp; @XmlAttribute(name = "name") @@ -48,7 +48,7 @@ public class SimpleField extends FieldBase implements Serializable, public SimpleField() {} - public SimpleField(String name, Map markings, String columnVisibility, Long timestamp, Object value) { + public SimpleField(String name, Markings markings, String columnVisibility, Long timestamp, Object value) { super(); this.name = name; this.markings = markings; @@ -134,17 +134,13 @@ public void setName(String name) { this.name = name; } - private void assureMarkings() { - if (this.markings == null) - this.markings = Maps.newHashMap(); - } - - public Map getMarkings() { - this.assureMarkings(); - return markings; + @Override + public Markings getMarkings() { + return this.markings; } - public void setMarkings(Map markings) { + @Override + public void setMarkings(Markings markings) { this.markings = markings; } @@ -198,7 +194,7 @@ public void writeTo(Output output, SimpleField message) throws IOException { if (message.value != null) output.writeObject(4, message.value, message.value.cachedSchema(), false); if (message.markings != null) { - output.writeObject(5, message.markings, MAP_SCHEMA, false); + output.writeObject(5, (AccessExpressionMarkings) message.markings, AccessExpressionMarkings.SCHEMA, false); } } @@ -220,10 +216,9 @@ public void mergeFrom(Input input, SimpleField message) throws IOException { message.value = input.mergeObject(null, TypedValue.getSchema()); break; case 5: - if (message.markings == null) - message.markings = Maps.newHashMap(); - Map markings = input.mergeObject(null, MAP_SCHEMA); - message.markings.putAll(markings); + message.markings = AccessExpressionMarkings.builder().build(); + input.mergeObject((AccessExpressionMarkings) message.markings, AccessExpressionMarkings.SCHEMA); + break; default: input.handleUnknownField(number, this); break; diff --git a/warehouse/query-core/src/test/java/datawave/query/predicate/ValueToAttributesTest.java b/warehouse/query-core/src/test/java/datawave/query/predicate/ValueToAttributesTest.java index dcc4db12af8..c5f7deb9e2e 100644 --- a/warehouse/query-core/src/test/java/datawave/query/predicate/ValueToAttributesTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/predicate/ValueToAttributesTest.java @@ -36,6 +36,7 @@ import datawave.core.query.configuration.GenericQueryConfiguration; import datawave.helpers.PrintUtility; import datawave.ingest.data.TypeRegistry; +import datawave.marking.AccessExpressionMarkings; import datawave.marking.MarkingFunctions; import datawave.microservice.query.QueryImpl; import datawave.query.QueryTestTableHelper; @@ -260,7 +261,7 @@ public void testComposites() { TypeMetadata typeMetadata = new TypeMetadata( "dts:[0:beep];types:[0:datawave.data.type.DateType,1:datawave.data.type.IpAddressType,2:datawave.data.type.LcNoDiacriticsType,3:datawave.data.type.NoOpType,4:datawave.data.type.NumberType];MAKE:[0:2];MAKE_COLOR:[0:3];START_DATE:[0:0];TYPE_NOEVAL:[0:2];IP_ADDR:[0:1];WHEELS:[0:2,0:4];COLOR:[0:2];COLOR_WHEELS:[0:3];TYPE:[0:2]"); - MarkingFunctions markingFunctions = new MarkingFunctions.Default(); + MarkingFunctions markingFunctions = new MarkingFunctions.Default(); ValueToAttributes valueToAttributes = new ValueToAttributes(compositeMetadata, typeMetadata, null, markingFunctions, true); } } diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/RemoteEventQueryLogicTest.java b/warehouse/query-core/src/test/java/datawave/query/tables/RemoteEventQueryLogicTest.java index 3a07901af58..2a9cf9cc9b9 100644 --- a/warehouse/query-core/src/test/java/datawave/query/tables/RemoteEventQueryLogicTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/tables/RemoteEventQueryLogicTest.java @@ -6,7 +6,6 @@ import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; import java.util.List; @@ -48,13 +47,13 @@ public void setup() { DefaultEventQueryResponse response1 = new DefaultEventQueryResponse(); DefaultEvent event1 = new DefaultEvent(); - event1.setFields(Collections.singletonList(new DefaultField("FOO1", "FOO|BAR", new HashMap(), -1L, "FOOBAR1"))); + event1.setFields(Collections.singletonList(new DefaultField("FOO1", "FOO|BAR", -1L, "FOOBAR1"))); response1.setEvents(Collections.singletonList(event1)); response1.setReturnedEvents(1L); DefaultEventQueryResponse response2 = new DefaultEventQueryResponse(); DefaultEvent event2 = new DefaultEvent(); - event1.setFields(Collections.singletonList(new DefaultField("FOO2", "FOO|BAR", new HashMap(), -1L, "FOOBAR2"))); + event1.setFields(Collections.singletonList(new DefaultField("FOO2", "FOO|BAR", -1L, "FOOBAR2"))); response2.setEvents(Collections.singletonList(event1)); response2.setReturnedEvents(1L); diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/RemoteQueryServiceTestUtil.java b/warehouse/query-core/src/test/java/datawave/query/tables/RemoteQueryServiceTestUtil.java index 5c7ad67954f..fd5763ebb41 100644 --- a/warehouse/query-core/src/test/java/datawave/query/tables/RemoteQueryServiceTestUtil.java +++ b/warehouse/query-core/src/test/java/datawave/query/tables/RemoteQueryServiceTestUtil.java @@ -23,7 +23,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Date; -import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.UUID; @@ -155,7 +154,7 @@ public void initialize() throws IOException { DefaultEventQueryResponse response = new DefaultEventQueryResponse(); if (nextCount < totalNext) { DefaultEvent event = new DefaultEvent(); - event.setFields(Collections.singletonList(new DefaultField("FOO" + nextCount, "FOO|BAR", new HashMap(), -1L, "FOOBAR" + nextCount))); + event.setFields(Collections.singletonList(new DefaultField("FOO" + nextCount, "FOO|BAR", -1L, "FOOBAR" + nextCount))); response.setEvents(Collections.singletonList(event)); response.setReturnedEvents(1L); } else { diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/content/ContentQueryLogicTest.java b/warehouse/query-core/src/test/java/datawave/query/tables/content/ContentQueryLogicTest.java index 6d83a2af5dd..35ec03c1f83 100644 --- a/warehouse/query-core/src/test/java/datawave/query/tables/content/ContentQueryLogicTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/tables/content/ContentQueryLogicTest.java @@ -55,7 +55,7 @@ public class ContentQueryLogicTest { private ContentQueryConfiguration mockContentConfig; private Query query; - private final MarkingFunctions markingFunctions = MarkingFunctionsFactory.createMarkingFunctions(); + private final MarkingFunctions markingFunctions = MarkingFunctionsFactory.createMarkingFunctions(); @BeforeEach public void setup() throws TableNotFoundException { diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/keyword/TagCloudTestUtil.java b/warehouse/query-core/src/test/java/datawave/query/tables/keyword/TagCloudTestUtil.java index c10197377fc..fd150779db1 100644 --- a/warehouse/query-core/src/test/java/datawave/query/tables/keyword/TagCloudTestUtil.java +++ b/warehouse/query-core/src/test/java/datawave/query/tables/keyword/TagCloudTestUtil.java @@ -4,6 +4,8 @@ import java.util.List; import java.util.Map; +import datawave.marking.AccessExpressionMarkings; +import datawave.marking.Markings; import datawave.query.tables.keyword.transform.KeywordResultsTransformer; import datawave.webservice.result.keyword.DefaultTagCloud; import datawave.webservice.result.keyword.DefaultTagCloudEntry; @@ -12,6 +14,9 @@ * Utility for producing TagClouds and supporting objects for tests */ public class TagCloudTestUtil { + + private static final Markings ALL_MARKINGS = AccessExpressionMarkings.builder().columnVisibility("ALL").build(); + public static final String SOPRANO_SOURCE = "20130101_0/test/-1kfeoq.-80b5fs.r0262j"; public static final String CAPONE_SOURCE = "20130101_0/test/-cvy0gj.tlf59s.-duxzua"; public static final String CORLEONE_SOURCE = "20130101_0/test/-d5uxna.msizfm.-oxy0iu"; @@ -27,7 +32,7 @@ public DefaultTagCloud getExpectedCloud(String version, Map metad } else { expectedCloud.setMetadata(metadata); } - expectedCloud.setMarkings(Map.of("visibility", "[ALL]")); + expectedCloud.setMarkings(ALL_MARKINGS); expectedCloud.setTags(entries); expectedCloud.setIntermediateResult(false); @@ -45,7 +50,7 @@ public DefaultTagCloud getCaponeKeywordCloud(String docId, String version) { entries.add(createTagCloudEntry("much farther", 0.5903, 1, List.of(docId))); DefaultTagCloud expectedCloud = new DefaultTagCloud(); - expectedCloud.setMarkings(Map.of("visibility", "[ALL]")); + expectedCloud.setMarkings(ALL_MARKINGS); expectedCloud.setTags(entries); if (version.equals("1")) { expectedCloud.setLanguage("keyword"); diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepIngestQueryTest.java b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepIngestQueryTest.java index 1e944d2749b..bb679633a3e 100644 --- a/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepIngestQueryTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepIngestQueryTest.java @@ -29,6 +29,7 @@ import datawave.core.query.result.event.DefaultResponseObjectFactory; import datawave.helpers.PrintUtility; import datawave.ingest.mapreduce.handler.ssdeep.SSDeepIndexHandler; +import datawave.marking.AccessExpressionMarkings; import datawave.marking.MarkingFunctions; import datawave.microservice.query.QueryImpl; import datawave.microservice.querymetric.QueryMetricFactoryImpl; @@ -87,7 +88,7 @@ public static void filterSetup() throws Exception { @Before public void setupQuery() { - MarkingFunctions markingFunctions = new MarkingFunctions.Default(); + MarkingFunctions markingFunctions = new MarkingFunctions.Default(); ResponseObjectFactory responseFactory = new DefaultResponseObjectFactory(); MetadataHelperFactory metadataHelperFactory = new MetadataHelperFactory(); diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/testframework/SSDeepDataType.java b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/testframework/SSDeepDataType.java index 2acb83e6943..4b935e6349e 100644 --- a/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/testframework/SSDeepDataType.java +++ b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/testframework/SSDeepDataType.java @@ -21,7 +21,6 @@ import datawave.ingest.input.reader.EventRecordReader; import datawave.ingest.mapreduce.handler.shard.AbstractColumnBasedHandler; import datawave.ingest.mapreduce.handler.ssdeep.SSDeepIndexHandler; -import datawave.marking.MarkingFunctions; import datawave.query.testframework.AbstractDataTypeConfig; import datawave.query.testframework.FieldConfig; import datawave.query.testframework.RawMetaData; @@ -247,7 +246,7 @@ public String getSecurityMarkingFieldNames() { @Override public String getSecurityMarkingFieldDomains() { - return MarkingFunctions.Default.COLUMN_VISIBILITY; + return "columnVisibility"; } @Override diff --git a/warehouse/query-core/src/test/java/datawave/query/testframework/CitiesDataType.java b/warehouse/query-core/src/test/java/datawave/query/testframework/CitiesDataType.java index 6111b298b0e..2d9efd43309 100644 --- a/warehouse/query-core/src/test/java/datawave/query/testframework/CitiesDataType.java +++ b/warehouse/query-core/src/test/java/datawave/query/testframework/CitiesDataType.java @@ -22,7 +22,6 @@ import datawave.ingest.data.config.CSVHelper; import datawave.ingest.data.config.ingest.BaseIngestHelper; import datawave.ingest.input.reader.EventRecordReader; -import datawave.marking.MarkingFunctions; /** * Contains all of the relevant data needed to configure any of the cities data types. @@ -256,7 +255,7 @@ public String getSecurityMarkingFieldNames() { @Override public String getSecurityMarkingFieldDomains() { - return MarkingFunctions.Default.COLUMN_VISIBILITY; + return "columnVisibility"; } @Override diff --git a/warehouse/query-core/src/test/java/datawave/query/testframework/cardata/CarsDataType.java b/warehouse/query-core/src/test/java/datawave/query/testframework/cardata/CarsDataType.java index c7adedc3016..9a66a903f21 100644 --- a/warehouse/query-core/src/test/java/datawave/query/testframework/cardata/CarsDataType.java +++ b/warehouse/query-core/src/test/java/datawave/query/testframework/cardata/CarsDataType.java @@ -20,7 +20,6 @@ import datawave.ingest.data.config.CSVHelper; import datawave.ingest.data.config.ingest.BaseIngestHelper; import datawave.ingest.input.reader.EventRecordReader; -import datawave.marking.MarkingFunctions; import datawave.query.testframework.AbstractDataTypeConfig; import datawave.query.testframework.FieldConfig; import datawave.query.testframework.RawMetaData; @@ -130,7 +129,7 @@ public static List eventSecurityFieldNames() { } public static List eventSecurityFieldDomains() { - return Collections.singletonList(MarkingFunctions.Default.COLUMN_VISIBILITY); + return Collections.singletonList("columnVisibility"); } /** diff --git a/warehouse/query-core/src/test/java/datawave/query/transformer/DocumentTransformerTest.java b/warehouse/query-core/src/test/java/datawave/query/transformer/DocumentTransformerTest.java index b80b565ef0f..ebc7f644bf1 100644 --- a/warehouse/query-core/src/test/java/datawave/query/transformer/DocumentTransformerTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/transformer/DocumentTransformerTest.java @@ -32,7 +32,7 @@ public class DocumentTransformerTest { // variables used to build the transformer private final Query query = new QueryImpl(); - private final MarkingFunctions markingFunctions = MarkingFunctionsFactory.createMarkingFunctions(); + private final MarkingFunctions markingFunctions = MarkingFunctionsFactory.createMarkingFunctions(); private final ResponseObjectFactory responseObjectFactory = new DefaultResponseObjectFactory(); private final KryoDocumentSerializer serializer = new KryoDocumentSerializer(); diff --git a/warehouse/query-core/src/test/java/datawave/query/transformer/FieldRenameTransformTest.java b/warehouse/query-core/src/test/java/datawave/query/transformer/FieldRenameTransformTest.java index d9e179e3153..347d1d7afeb 100644 --- a/warehouse/query-core/src/test/java/datawave/query/transformer/FieldRenameTransformTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/transformer/FieldRenameTransformTest.java @@ -12,7 +12,6 @@ import org.apache.commons.collections.keyvalue.UnmodifiableMapEntry; import org.junit.Test; -import datawave.marking.MarkingFunctions; import datawave.query.Constants; import datawave.query.attributes.Attributes; import datawave.query.attributes.Document; @@ -21,7 +20,7 @@ public class FieldRenameTransformTest { @Test - public void renameIdentityMapTest() throws MarkingFunctions.Exception { + public void renameIdentityMapTest() { Key key = new Key("shard", "dataType" + Constants.NULL + "uid"); Set fieldMap = new HashSet<>(); @@ -44,7 +43,7 @@ public void renameIdentityMapTest() throws MarkingFunctions.Exception { } @Test - public void renameFieldMapTest() throws MarkingFunctions.Exception { + public void renameFieldMapTest() { Key key = new Key("shard", "dataType" + Constants.NULL + "uid"); Set fieldMap = new HashSet<>(); @@ -70,7 +69,7 @@ public void renameFieldMapTest() throws MarkingFunctions.Exception { } @Test - public void renameFieldMapPreexistingTest() throws MarkingFunctions.Exception { + public void renameFieldMapPreexistingTest() { Key key = new Key("shard", "dataType" + Constants.NULL + "uid"); Set fieldMap = new HashSet<>(); @@ -96,7 +95,7 @@ public void renameFieldMapPreexistingTest() throws MarkingFunctions.Exception { } @Test - public void renameWithGroupingContextAndMultipleMappings() throws MarkingFunctions.Exception { + public void renameWithGroupingContextAndMultipleMappings() { Key key = new Key("shard", "dataType" + Constants.NULL + "uid"); Set fieldMap = new HashSet<>(); diff --git a/warehouse/query-core/src/test/java/datawave/query/transformer/GroupingTest.java b/warehouse/query-core/src/test/java/datawave/query/transformer/GroupingTest.java index 75fe059fb01..2769d25bb16 100644 --- a/warehouse/query-core/src/test/java/datawave/query/transformer/GroupingTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/transformer/GroupingTest.java @@ -11,6 +11,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.SortedMap; import java.util.SortedSet; @@ -44,6 +45,7 @@ import datawave.core.query.iterator.DatawaveTransformIterator; import datawave.helpers.PrintUtility; import datawave.ingest.data.TypeRegistry; +import datawave.marking.Markings; import datawave.query.QueryParameters; import datawave.query.QueryTestTableHelper; import datawave.query.RebuildingScannerTestHelper; @@ -604,13 +606,15 @@ public void testGroupByWithReducedResponse() throws Exception { for (QueryResult result : queryResults) { // noinspection rawtypes for (EventBase event : result.response.getEvents()) { - String eventCV = event.getMarkings().get(COLVIS_MARKING).toString(); + String eventCV = Optional.ofNullable(event.getMarkings()).map(Markings::getMarkings).map(Object::toString).orElse(null); assertThat(eventCV).describedAs("Assert event cv for teardown: %s, interrupt: %s", result.teardown, result.interrupt).isEqualTo(REDUCED_COLVIS); // noinspection unchecked for (FieldBase field : (List>) event.getFields()) { - String fieldCV = field.getMarkings().get(COLVIS_MARKING); - assertThat(fieldCV).describedAs("Assert null field cv for field: %s, teardown: %s, interrupt: %s", field.getName(), result.teardown, - result.interrupt).isNull(); + Markings fieldMarkings = field.getMarkings(); + assertThat(fieldMarkings == null || fieldMarkings.isEmpty()) + .describedAs("Assert empty field markings for field: %s, teardown: %s, interrupt: %s", field.getName(), result.teardown, + result.interrupt) + .isTrue(); } } } diff --git a/warehouse/query-core/src/test/java/datawave/query/transformer/annotation/AnnotationHitsTransformerTest.java b/warehouse/query-core/src/test/java/datawave/query/transformer/annotation/AnnotationHitsTransformerTest.java index 2a2a88c152b..077bae65f68 100644 --- a/warehouse/query-core/src/test/java/datawave/query/transformer/annotation/AnnotationHitsTransformerTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/transformer/annotation/AnnotationHitsTransformerTest.java @@ -42,6 +42,7 @@ import datawave.annotation.protobuf.v1.SegmentBoundary; import datawave.annotation.protobuf.v1.SegmentValue; import datawave.data.normalizer.Normalizer; +import datawave.marking.AccessExpressionMarkings; import datawave.marking.MarkingFunctions; import datawave.microservice.query.Query; import datawave.microservice.query.QueryImpl; @@ -117,7 +118,7 @@ public class AnnotationHitsTransformerTest { private ShardQueryConfiguration shardQueryConfiguration; private Query settings; - private MarkingFunctions markingFunctions; + private MarkingFunctions markingFunctions; private List annotations; private AnnotationSource annotationSource; diff --git a/warehouse/query-core/src/test/java/datawave/test/GroupAssert.java b/warehouse/query-core/src/test/java/datawave/test/GroupAssert.java index 23c4c7c2c29..e90693f070d 100644 --- a/warehouse/query-core/src/test/java/datawave/test/GroupAssert.java +++ b/warehouse/query-core/src/test/java/datawave/test/GroupAssert.java @@ -36,8 +36,8 @@ public GroupAssert hasCount(int count) { public GroupAssert hasVisibilitiesForKey(GroupingAttribute key, ColumnVisibility... visibilities) { isNotNull(); - Collection actualVisibilities = actual.getVisibilitiesForAttribute(key); - Assertions.assertThat(actualVisibilities).isNotNull().withFailMessage("Expected column visibilties for %s to contain exactly %s but was %s", key, + Collection actualVisibilities = actual.getColumnVisibilitiesForAttribute(key); + Assertions.assertThat(actualVisibilities).isNotNull().withFailMessage("Expected access expressions for %s to contain exactly %s but was %s", key, Arrays.toString(visibilities), actualVisibilities).containsExactlyInAnyOrder(visibilities); return this; } @@ -46,7 +46,7 @@ public GroupAssert hasDocumentVisibilities(ColumnVisibility... visibilities) { isNotNull(); Collection actualVisibilities = actual.getDocumentVisibilities(); Assertions.assertThat(actualVisibilities).isNotNull() - .withFailMessage("Expected document visibilities to contain exactly %s but was %s", Arrays.toString(visibilities), actualVisibilities) + .withFailMessage("Expected document expressions to contain exactly %s but was %s", Arrays.toString(visibilities), actualVisibilities) .containsExactlyInAnyOrder(visibilities); return this; } diff --git a/web-services/cached-results/src/main/java/datawave/webservice/results/cached/CachedResultsBean.java b/web-services/cached-results/src/main/java/datawave/webservice/results/cached/CachedResultsBean.java index f416641273e..cef2d1a00a6 100644 --- a/web-services/cached-results/src/main/java/datawave/webservice/results/cached/CachedResultsBean.java +++ b/web-services/cached-results/src/main/java/datawave/webservice/results/cached/CachedResultsBean.java @@ -26,7 +26,6 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; -import java.util.TreeMap; import java.util.UUID; import java.util.concurrent.Future; @@ -63,6 +62,7 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; +import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.commons.collections4.Transformer; import org.apache.commons.dbutils.DbUtils; @@ -94,7 +94,6 @@ import datawave.core.query.predict.QueryPredictor; import datawave.interceptor.RequiredInterceptor; import datawave.interceptor.ResponseInterceptor; -import datawave.marking.MarkingFunctions; import datawave.marking.SecurityMarking; import datawave.microservice.query.Query; import datawave.microservice.query.QueryParameters; @@ -303,7 +302,16 @@ protected void loadBatch(PreparedStatement ps, String owner, String queryId, Str ps.setString(5, cqo.getEventId()); ps.setString(6, cqo.getRow()); ps.setString(7, cqo.getColFam()); - ps.setString(8, MarkingFunctions.Encoding.toString(new TreeMap<>(cqo.getMarkings()))); + + String visibility = ""; + if (cqo.getMarkings() != null) { + AccessExpression ae = cqo.getMarkings().toAccessExpression(); + if (ae != null) { + visibility = ae.getExpression(); + } + } + + ps.setString(8, visibility); for (Entry e : cqo.getColumnValues().entrySet()) { String columnName = e.getKey(); @@ -475,7 +483,7 @@ protected GenericResponse load(@Required("queryId") String queryId, Stri try { MultiValueMap queryMap = new LinkedMultiValueMap<>(q.toMap()); marking.validate(queryMap); - queryMap.set(PrivateAuditConstants.COLUMN_VISIBILITY, marking.toColumnVisibilityString()); + queryMap.set(PrivateAuditConstants.COLUMN_VISIBILITY, marking.toAccessExpressionString()); queryMap.set(PrivateAuditConstants.AUDIT_TYPE, auditType.name()); queryMap.set(PrivateAuditConstants.USER_DN, q.getUserDN()); queryMap.set(PrivateAuditConstants.LOGIC_CLASS, logic.getLogicName()); @@ -1310,7 +1318,7 @@ public CachedResultsResponse create(@Required("queryId") @PathParam("queryId") S MultiValueMap params = new LinkedMultiValueMap<>(query.toMap()); marking.validate(params); PrivateAuditConstants.stripPrivateParameters(queryParameters); - params.set(PrivateAuditConstants.COLUMN_VISIBILITY, marking.toColumnVisibilityString()); + params.set(PrivateAuditConstants.COLUMN_VISIBILITY, marking.toAccessExpressionString()); params.set(PrivateAuditConstants.AUDIT_TYPE, auditType.name()); params.set(PrivateAuditConstants.USER_DN, query.getUserDN()); params.set(PrivateAuditConstants.LOGIC_CLASS, crq.getQueryLogic().getLogicName()); diff --git a/web-services/client/src/main/java/datawave/webservice/modification/DefaultModificationRequest.java b/web-services/client/src/main/java/datawave/webservice/modification/DefaultModificationRequest.java index d1933578cbe..542c764e325 100644 --- a/web-services/client/src/main/java/datawave/webservice/modification/DefaultModificationRequest.java +++ b/web-services/client/src/main/java/datawave/webservice/modification/DefaultModificationRequest.java @@ -84,7 +84,7 @@ public Map getFieldMarkings() { } public void setFieldMarkings(Map fieldMarkings) { - this.fieldMarkings = (fieldMarkings == null ? new HashMap() : new HashMap(fieldMarkings)); + this.fieldMarkings = (fieldMarkings == null ? new HashMap<>() : new HashMap<>(fieldMarkings)); } public Map getOldFieldMarkings() { @@ -94,7 +94,7 @@ public Map getOldFieldMarkings() { } public void setOldFieldMarkings(Map oldFieldMarkings) { - this.oldFieldMarkings = (oldFieldMarkings == null ? new HashMap() : new HashMap(oldFieldMarkings)); + this.oldFieldMarkings = (oldFieldMarkings == null ? new HashMap<>() : new HashMap<>(oldFieldMarkings)); } public String getOldFieldValue() { diff --git a/web-services/client/src/main/java/datawave/webservice/modification/ModificationRequestBase.java b/web-services/client/src/main/java/datawave/webservice/modification/ModificationRequestBase.java index d52c9e16201..f2db7ad4e56 100644 --- a/web-services/client/src/main/java/datawave/webservice/modification/ModificationRequestBase.java +++ b/web-services/client/src/main/java/datawave/webservice/modification/ModificationRequestBase.java @@ -72,7 +72,7 @@ public static class DefaultFieldMarkingsAdapter extends XmlAdapter unmarshal(DefaultFieldMarking value) throws Exception { - HashMap fieldMarkings = new HashMap(); + HashMap fieldMarkings = new HashMap<>(); fieldMarkings.put(COLUMN_VISIBILITY, value.fieldColumnVisibility); return fieldMarkings; } diff --git a/web-services/client/src/main/java/datawave/webservice/query/cachedresults/CacheableQueryRow.java b/web-services/client/src/main/java/datawave/webservice/query/cachedresults/CacheableQueryRow.java index b3157341f62..d01668bc0f9 100644 --- a/web-services/client/src/main/java/datawave/webservice/query/cachedresults/CacheableQueryRow.java +++ b/web-services/client/src/main/java/datawave/webservice/query/cachedresults/CacheableQueryRow.java @@ -8,6 +8,7 @@ import java.util.Set; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.webservice.query.result.event.HasMarkings; import datawave.webservice.query.util.TypedValue; @@ -27,15 +28,15 @@ public abstract class CacheableQueryRow implements HasMarkings { fixedColumnSet.add("_column_timestamps_"); } - protected MarkingFunctions markingFunctions; + protected MarkingFunctions markingFunctions; public static Set getFixedColumnSet() { return Collections.unmodifiableSet(fixedColumnSet); } - public abstract void addColumn(String columnName, String columnStringValue, Map markings, String columnVisibility, Long timestamp); + public abstract void addColumn(String columnName, String columnStringValue, Markings markings, String columnVisibility, Long timestamp); - public abstract void addColumn(String columnName, TypedValue columnValue, Map markings, String columnVisibility, Long timestamp); + public abstract void addColumn(String columnName, TypedValue columnValue, Markings markings, String columnVisibility, Long timestamp); public abstract String getUser(); @@ -51,7 +52,7 @@ public static Set getFixedColumnSet() { public abstract String getColFam(); - public abstract Map getColumnMarkings(String columnName); + public abstract Markings getColumnMarkings(String columnName); public abstract String getColumnVisibility(String columnName); @@ -98,17 +99,17 @@ protected static String createColumnList(Set columnNames, Map> columnMarkingsMap); + public abstract void setColumnMarkingsMap(Map> columnMarkingsMap); public abstract void setColumnColumnVisibilityMap(Map columnVisibilityMap); public abstract void setColumnTimestampMap(Map parseColumnTimestamps); - public MarkingFunctions getMarkingFunctions() { + public MarkingFunctions getMarkingFunctions() { return markingFunctions; } - public void setMarkingFunctions(MarkingFunctions markingFunctions) { + public void setMarkingFunctions(MarkingFunctions markingFunctions) { this.markingFunctions = markingFunctions; } } diff --git a/web-services/client/src/main/java/datawave/webservice/query/result/EdgeQueryResponseBase.java b/web-services/client/src/main/java/datawave/webservice/query/result/EdgeQueryResponseBase.java index ac2c03acbe9..8419d0664e8 100644 --- a/web-services/client/src/main/java/datawave/webservice/query/result/EdgeQueryResponseBase.java +++ b/web-services/client/src/main/java/datawave/webservice/query/result/EdgeQueryResponseBase.java @@ -1,11 +1,11 @@ package datawave.webservice.query.result; import java.util.List; -import java.util.Map; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; +import datawave.marking.Markings; import datawave.webservice.query.result.edge.EdgeBase; import datawave.webservice.query.result.event.HasMarkings; import datawave.webservice.result.BaseQueryResponse; @@ -13,7 +13,7 @@ @XmlAccessorType(XmlAccessType.NONE) public abstract class EdgeQueryResponseBase extends BaseQueryResponse implements HasMarkings { - protected Map markings; + protected Markings markings; private static final long serialVersionUID = 5957852042933196910L; diff --git a/web-services/client/src/main/java/datawave/webservice/query/result/edge/DefaultEdge.java b/web-services/client/src/main/java/datawave/webservice/query/result/edge/DefaultEdge.java index d535e7009e9..357ea392559 100644 --- a/web-services/client/src/main/java/datawave/webservice/query/result/edge/DefaultEdge.java +++ b/web-services/client/src/main/java/datawave/webservice/query/result/edge/DefaultEdge.java @@ -5,7 +5,6 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; -import java.util.Map; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -17,6 +16,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; +import datawave.marking.Markings; import datawave.webservice.query.util.OptionallyEncodedStringAdapter; import io.protostuff.Input; import io.protostuff.Message; @@ -30,7 +30,7 @@ public class DefaultEdge implements EdgeBase, Serializable, Message private static final long serialVersionUID = -1621626861385080614L; - protected Map markings; + protected Markings markings; // The visibility of this edge @XmlElement(name = "ColumnVisibility", required = true, nillable = false) @@ -208,12 +208,12 @@ public void setActivityDate(String activityDate) { } @Override - public void setMarkings(Map markings) { + public void setMarkings(Markings markings) { this.markings = markings; - this.setColumnVisibility(markings.get(COLUMN_VISIBILITY)); } - public Map getMarkings() { + @Override + public Markings getMarkings() { return this.markings; } diff --git a/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultEvent.java b/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultEvent.java index 5b535e044be..837c8d8d38f 100644 --- a/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultEvent.java +++ b/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultEvent.java @@ -5,7 +5,6 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; -import java.util.Map; import javax.xml.bind.annotation.XmlAccessOrder; import javax.xml.bind.annotation.XmlAccessType; @@ -14,12 +13,12 @@ import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlElementWrapper; import javax.xml.bind.annotation.XmlTransient; -import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; import org.apache.commons.lang.builder.EqualsBuilder; +import datawave.marking.AccessExpressionMarkings; +import datawave.marking.Markings; import datawave.webservice.query.data.ObjectSizeOf; -import datawave.webservice.xml.util.StringMapAdapter; import io.protostuff.Input; import io.protostuff.Message; import io.protostuff.Output; @@ -31,16 +30,15 @@ public class DefaultEvent extends EventBase implement private static final long serialVersionUID = 2390592143914560317L; - @XmlElement(name = "Markings") - @XmlJavaTypeAdapter(StringMapAdapter.class) - private HashMap markings = null; + @XmlElement(name = "Markings", type = AccessExpressionMarkings.class) + private Markings markings; @XmlElement(name = "Metadata") - private Metadata metadata = null; + private Metadata metadata; @XmlElementWrapper(name = "Fields") @XmlElement(name = "Field") - private List fields = null; + private List fields; public List getFields() { return fields; @@ -56,21 +54,13 @@ public String toString() { } @Override - public Map getMarkings() { - if (markings != null) { - return markings; - } else { - return super.getMarkings(); - } + public Markings getMarkings() { + return this.markings; } - public void setMarkings(Map markings) { - if (null != markings) { - this.markings = new HashMap<>(markings); - } else { - this.markings = null; - } - super.setMarkings(this.markings); + @Override + public void setMarkings(Markings markings) { + this.markings = markings; } public Metadata getMetadata() { @@ -134,7 +124,7 @@ public boolean isInitialized(DefaultEvent message) { public void writeTo(Output output, DefaultEvent message) throws IOException { if (message.markings != null) - output.writeObject(1, message.markings, MapSchema.SCHEMA, false); + output.writeObject(1, (AccessExpressionMarkings) message.markings, AccessExpressionMarkings.SCHEMA, false); if (message.metadata != null) { output.writeObject(2, message.metadata, Metadata.getSchema(), false); @@ -159,8 +149,8 @@ public void mergeFrom(Input input, DefaultEvent message) throws IOException { while ((number = input.readFieldNumber(this)) != 0) { switch (number) { case 1: - message.markings = new HashMap(); - input.mergeObject(message.markings, MapSchema.SCHEMA); + message.markings = AccessExpressionMarkings.builder().build(); + input.mergeObject((AccessExpressionMarkings) message.markings, AccessExpressionMarkings.SCHEMA); break; case 2: message.metadata = input.mergeObject(null, Metadata.getSchema()); @@ -197,7 +187,7 @@ public String getFieldName(int number) { public int getFieldNumber(String name) { final Integer number = fieldMap.get(name); - return number == null ? 0 : number.intValue(); + return number == null ? 0 : number; } final HashMap fieldMap = new HashMap(); diff --git a/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultFacets.java b/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultFacets.java index b0a1416b85b..b11be50cf80 100644 --- a/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultFacets.java +++ b/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultFacets.java @@ -4,7 +4,6 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; -import java.util.Map; import javax.xml.bind.annotation.XmlAccessOrder; import javax.xml.bind.annotation.XmlAccessType; @@ -12,6 +11,7 @@ import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlTransient; +import datawave.marking.Markings; import datawave.webservice.query.data.ObjectSizeOf; import io.protostuff.Input; import io.protostuff.Message; @@ -24,11 +24,13 @@ public class DefaultFacets extends FacetsBase implements Serializable, Message markings) { + @Override + public void setMarkings(Markings markings) { this.markings = markings; } - public Map getMarkings() { + @Override + public Markings getMarkings() { return this.markings; } diff --git a/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultField.java b/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultField.java index 86c78a21b7e..0acf4b41fc3 100644 --- a/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultField.java +++ b/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultField.java @@ -3,7 +3,6 @@ import java.io.IOException; import java.io.Serializable; import java.util.HashMap; -import java.util.Map; import javax.xml.bind.annotation.XmlAccessOrder; import javax.xml.bind.annotation.XmlAccessType; @@ -12,7 +11,6 @@ import javax.xml.bind.annotation.XmlAttribute; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlTransient; -import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; @@ -20,8 +18,9 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import datawave.data.type.Type; +import datawave.marking.AccessExpressionMarkings; +import datawave.marking.Markings; import datawave.webservice.query.util.TypedValue; -import datawave.webservice.xml.util.StringMapAdapter; import io.protostuff.Input; import io.protostuff.Message; import io.protostuff.Output; @@ -38,9 +37,8 @@ public class DefaultField extends FieldBase implements Serializabl private static final long serialVersionUID = -3982566563059126017L; - @XmlElement(name = "Markings") - @XmlJavaTypeAdapter(StringMapAdapter.class) - private HashMap markings; + @XmlElement(name = "Markings", type = AccessExpressionMarkings.class) + private Markings markings; @XmlAttribute(name = "columnVisibility") private String columnVisibility; @XmlAttribute(name = "timestamp") @@ -52,10 +50,10 @@ public class DefaultField extends FieldBase implements Serializabl public DefaultField() {} - public DefaultField(String name, String columnVisibility, Map markings, Long timestamp, Object value) { + public DefaultField(String name, String columnVisibility, Markings markings, Long timestamp, Object value) { super(); this.name = name; - this.markings = new HashMap(markings); + this.markings = markings; this.columnVisibility = columnVisibility; this.timestamp = timestamp; this.value = new TypedValue(value); @@ -69,17 +67,13 @@ public DefaultField(String name, String columnVisibility, Long timestamp, Object this.value = new TypedValue(value); } - /* - * (non-Javadoc) - * - * @see datawave.webservice.query.result.event.FieldInterface#setMarkings(java.util.Map) - */ @Override - public void setMarkings(Map markings) { - this.markings = new HashMap(markings); + public void setMarkings(Markings markings) { + this.markings = markings; } - public Map getMarkings() { + @Override + public Markings getMarkings() { return markings; } @@ -216,7 +210,7 @@ public boolean isInitialized(DefaultField message) { @Override public void writeTo(Output output, DefaultField message) throws IOException { if (message.markings != null) - output.writeObject(1, message.markings, MapSchema.SCHEMA, false); + output.writeObject(1, (AccessExpressionMarkings) message.markings, AccessExpressionMarkings.SCHEMA, false); if (message.columnVisibility != null) output.writeString(2, message.columnVisibility, false); output.writeUInt64(3, message.timestamp, false); @@ -232,8 +226,8 @@ public void mergeFrom(Input input, DefaultField message) throws IOException { while ((number = input.readFieldNumber(this)) != 0) { switch (number) { case 1: - message.markings = new HashMap(); - input.mergeObject(message.markings, MapSchema.SCHEMA); + message.markings = AccessExpressionMarkings.builder().build(); + input.mergeObject((AccessExpressionMarkings) message.markings, AccessExpressionMarkings.SCHEMA); break; case 2: message.columnVisibility = input.readString(); diff --git a/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultFieldCardinality.java b/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultFieldCardinality.java index acb724af967..df5557c61fd 100644 --- a/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultFieldCardinality.java +++ b/web-services/client/src/main/java/datawave/webservice/query/result/event/DefaultFieldCardinality.java @@ -3,7 +3,6 @@ import java.io.IOException; import java.io.Serializable; import java.util.HashMap; -import java.util.Map; import javax.xml.bind.annotation.XmlAccessOrder; import javax.xml.bind.annotation.XmlAccessType; @@ -16,6 +15,7 @@ import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; +import datawave.marking.Markings; import io.protostuff.Input; import io.protostuff.Message; import io.protostuff.Output; @@ -65,11 +65,12 @@ public void setField(String field) { } @Override - public void setMarkings(Map markings) { + public void setMarkings(Markings markings) { this.markings = markings; } - public Map getMarkings() { + @Override + public Markings getMarkings() { return this.markings; } diff --git a/web-services/client/src/main/java/datawave/webservice/query/result/event/EventBase.java b/web-services/client/src/main/java/datawave/webservice/query/result/event/EventBase.java index 5314c8f1b0a..ce367657cd0 100644 --- a/web-services/client/src/main/java/datawave/webservice/query/result/event/EventBase.java +++ b/web-services/client/src/main/java/datawave/webservice/query/result/event/EventBase.java @@ -1,22 +1,17 @@ package datawave.webservice.query.result.event; import java.util.List; -import java.util.Map; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlSeeAlso; -import javax.xml.bind.annotation.XmlTransient; - -import com.google.common.collect.Maps; +import datawave.marking.Markings; import io.protostuff.Message; @XmlAccessorType(XmlAccessType.NONE) -@XmlSeeAlso(DefaultEvent.class) public abstract class EventBase> implements HasMarkings, Message { - protected transient Map markings; + protected transient Markings markings; protected transient boolean intermediateResult; @@ -47,20 +42,13 @@ public abstract class EventBase> implements HasMarkings public abstract void setFields(List fields); - @XmlTransient public abstract List getFields(); - public Map getMarkings() { - assureMarkings(); + public Markings getMarkings() { return markings; } - protected void assureMarkings() { - if (this.markings == null) - this.markings = Maps.newHashMap(); - } - - public void setMarkings(Map markings) { + public void setMarkings(Markings markings) { this.markings = markings; } diff --git a/web-services/client/src/main/java/datawave/webservice/query/result/event/FacetsBase.java b/web-services/client/src/main/java/datawave/webservice/query/result/event/FacetsBase.java index d5f6423dc4d..b01735a7f38 100644 --- a/web-services/client/src/main/java/datawave/webservice/query/result/event/FacetsBase.java +++ b/web-services/client/src/main/java/datawave/webservice/query/result/event/FacetsBase.java @@ -1,7 +1,6 @@ package datawave.webservice.query.result.event; import java.util.List; -import java.util.Map; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -10,7 +9,9 @@ import javax.xml.bind.annotation.XmlSeeAlso; import javax.xml.bind.annotation.XmlTransient; -import org.apache.accumulo.core.security.ColumnVisibility; +import org.apache.accumulo.access.AccessExpression; + +import datawave.marking.Markings; /** * @@ -19,14 +20,14 @@ @XmlSeeAlso(DefaultFacets.class) public abstract class FacetsBase implements HasMarkings { - protected Map markings; + protected Markings markings; @XmlElementWrapper(name = "Fields") @XmlElement(name = "Field") protected List fields = null; @XmlTransient - protected ColumnVisibility columnVisibility; + protected AccessExpression accessExpression; @SuppressWarnings("unchecked") public void setFields(List fields) { @@ -37,12 +38,12 @@ public List getFields() { return fields; } - public ColumnVisibility getColumnVisibility() { - return columnVisibility; + public AccessExpression getAccessExpression() { + return accessExpression; } - public void setColumnVisibility(ColumnVisibility columnVisibility) { - this.columnVisibility = columnVisibility; + public void setAccessExpression(AccessExpression accessExpression) { + this.accessExpression = accessExpression; } public abstract void setSizeInBytes(long sizeInBytes); diff --git a/web-services/client/src/main/java/datawave/webservice/query/result/event/FieldBase.java b/web-services/client/src/main/java/datawave/webservice/query/result/event/FieldBase.java index ad2fb2d0dfe..f8883f6dcd0 100644 --- a/web-services/client/src/main/java/datawave/webservice/query/result/event/FieldBase.java +++ b/web-services/client/src/main/java/datawave/webservice/query/result/event/FieldBase.java @@ -1,23 +1,21 @@ package datawave.webservice.query.result.event; -import java.util.Map; +import java.nio.charset.StandardCharsets; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlSeeAlso; +import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.security.ColumnVisibility; -import com.google.common.base.Charsets; - +import datawave.marking.Markings; import datawave.webservice.query.util.TypedValue; import io.protostuff.Message; @XmlAccessorType(XmlAccessType.NONE) -@XmlSeeAlso(DefaultField.class) public abstract class FieldBase implements HasMarkings, Message { - protected transient Map markings; + protected transient Markings markings; public abstract Long getTimestamp(); @@ -40,8 +38,13 @@ public abstract class FieldBase implements HasMarkings, Message { public abstract void setColumnVisibility(String columnVisibility); public void setColumnVisibility(ColumnVisibility columnVisibility) { - String cvString = (columnVisibility == null) ? null : new String(columnVisibility.getExpression(), Charsets.UTF_8); - setColumnVisibility(cvString); + String exprString = (columnVisibility == null) ? null : new String(columnVisibility.getExpression(), StandardCharsets.UTF_8); + setColumnVisibility(exprString); + } + + public void setColumnVisibility(AccessExpression accessExpression) { + String exprString = (accessExpression == null) ? null : accessExpression.getExpression(); + setColumnVisibility(exprString); } } diff --git a/web-services/client/src/main/java/datawave/webservice/query/result/event/FieldCardinalityBase.java b/web-services/client/src/main/java/datawave/webservice/query/result/event/FieldCardinalityBase.java index 62ca15fed8c..cb14e3eae84 100644 --- a/web-services/client/src/main/java/datawave/webservice/query/result/event/FieldCardinalityBase.java +++ b/web-services/client/src/main/java/datawave/webservice/query/result/event/FieldCardinalityBase.java @@ -1,14 +1,15 @@ package datawave.webservice.query.result.event; -import java.util.Map; +import java.nio.charset.StandardCharsets; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlSeeAlso; +import org.apache.accumulo.access.AccessExpression; import org.apache.accumulo.core.security.ColumnVisibility; -import com.google.common.base.Charsets; +import datawave.marking.Markings; /** * @@ -17,15 +18,20 @@ @XmlSeeAlso(DefaultFieldCardinality.class) public abstract class FieldCardinalityBase implements HasMarkings { - protected Map markings; + protected Markings markings; public abstract String getColumnVisibility(); public abstract void setColumnVisibility(String columnVisibility); public void setColumnVisibility(ColumnVisibility columnVisibility) { - String cvString = (columnVisibility == null) ? null : new String(columnVisibility.getExpression(), Charsets.UTF_8); - setColumnVisibility(cvString); + String exprString = (columnVisibility == null) ? null : new String(columnVisibility.getExpression(), StandardCharsets.UTF_8); + setColumnVisibility(exprString); + } + + public void setColumnVisibility(AccessExpression accessExpression) { + String exprString = (accessExpression == null) ? null : accessExpression.getExpression(); + setColumnVisibility(exprString); } public abstract String getField(); diff --git a/web-services/client/src/main/java/datawave/webservice/result/DefaultEdgeQueryResponse.java b/web-services/client/src/main/java/datawave/webservice/result/DefaultEdgeQueryResponse.java index e54d29659a9..04db73661b2 100644 --- a/web-services/client/src/main/java/datawave/webservice/result/DefaultEdgeQueryResponse.java +++ b/web-services/client/src/main/java/datawave/webservice/result/DefaultEdgeQueryResponse.java @@ -6,7 +6,6 @@ import java.util.Collections; import java.util.LinkedList; import java.util.List; -import java.util.Map; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -15,6 +14,7 @@ import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlTransient; +import datawave.marking.Markings; import datawave.webservice.query.exception.QueryExceptionType; import datawave.webservice.query.result.EdgeQueryResponseBase; import datawave.webservice.query.result.edge.DefaultEdge; @@ -53,12 +53,13 @@ public String getSecurityMarkings() { return securityMarkings; } - public void setMarkings(Map markings) { + @Override + public void setMarkings(Markings markings) { this.markings = markings; - this.setSecurityMarkings(markings.get("security")); } - public Map getMarkings() { + @Override + public Markings getMarkings() { return markings; } diff --git a/web-services/client/src/main/java/datawave/webservice/result/FacetQueryResponse.java b/web-services/client/src/main/java/datawave/webservice/result/FacetQueryResponse.java index 82da40f2863..c69149202f7 100644 --- a/web-services/client/src/main/java/datawave/webservice/result/FacetQueryResponse.java +++ b/web-services/client/src/main/java/datawave/webservice/result/FacetQueryResponse.java @@ -6,7 +6,6 @@ import java.util.Collections; import java.util.LinkedList; import java.util.List; -import java.util.Map; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -16,6 +15,7 @@ import javax.xml.bind.annotation.XmlTransient; import javax.xml.bind.annotation.XmlType; +import datawave.marking.Markings; import datawave.webservice.query.exception.QueryExceptionType; import datawave.webservice.query.result.event.DefaultFacets; import datawave.webservice.query.result.event.FacetsBase; @@ -46,11 +46,13 @@ public FacetQueryResponse(List facets) { this.facets = new ArrayList(facets); } - public void setMarkings(Map markings) { + @Override + public void setMarkings(Markings markings) { this.markings = markings; } - public Map getMarkings() { + @Override + public Markings getMarkings() { return markings; } diff --git a/web-services/client/src/main/java/datawave/webservice/result/FacetQueryResponseBase.java b/web-services/client/src/main/java/datawave/webservice/result/FacetQueryResponseBase.java index 27426102282..bbe2a288b40 100644 --- a/web-services/client/src/main/java/datawave/webservice/result/FacetQueryResponseBase.java +++ b/web-services/client/src/main/java/datawave/webservice/result/FacetQueryResponseBase.java @@ -1,11 +1,11 @@ package datawave.webservice.result; import java.util.List; -import java.util.Map; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; +import datawave.marking.Markings; import datawave.webservice.query.result.event.FacetsBase; import datawave.webservice.query.result.event.HasMarkings; @@ -16,7 +16,7 @@ public abstract class FacetQueryResponseBase extends BaseQueryResponse implements HasMarkings { private static final long serialVersionUID = -3483112784845232037L; - protected transient Map markings; + protected transient Markings markings; public abstract Long getTotalEvents(); @@ -27,6 +27,4 @@ public abstract class FacetQueryResponseBase extends BaseQueryResponse implement public abstract void setFacets(List facets); public abstract void addFacet(FacetsBase facetInterface); - - public abstract void setMarkings(Map markings); } diff --git a/web-services/client/src/main/java/datawave/webservice/result/keyword/DefaultTagCloud.java b/web-services/client/src/main/java/datawave/webservice/result/keyword/DefaultTagCloud.java index 4b20f2f12de..10561fdc2e5 100644 --- a/web-services/client/src/main/java/datawave/webservice/result/keyword/DefaultTagCloud.java +++ b/web-services/client/src/main/java/datawave/webservice/result/keyword/DefaultTagCloud.java @@ -18,6 +18,8 @@ import javax.xml.bind.annotation.XmlTransient; import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; +import datawave.marking.AccessExpressionMarkings; +import datawave.marking.Markings; import datawave.webservice.query.result.event.MapSchema; import datawave.webservice.xml.util.StringMapAdapter; import io.protostuff.Input; @@ -32,8 +34,7 @@ public class DefaultTagCloud extends TagCloudBase markings = null; + private Markings markings; @XmlElement(name = "language") private String language = null; @@ -47,21 +48,13 @@ public class DefaultTagCloud extends TagCloudBase tags = null; @Override - public Map getMarkings() { - if (markings != null) { - return markings; - } else { - return super.getMarkings(); - } + public Markings getMarkings() { + return markings; } - public void setMarkings(Map markings) { - if (null != markings) { - this.markings = new HashMap<>(markings); - } else { - this.markings = null; - } - super.setMarkings(this.markings); + @Override + public void setMarkings(Markings markings) { + this.markings = markings; } @Override @@ -121,7 +114,7 @@ public boolean isInitialized(DefaultTagCloud message) { public void writeTo(Output output, DefaultTagCloud message) throws IOException { if (message.markings != null) - output.writeObject(1, message.markings, MapSchema.SCHEMA, false); + output.writeObject(1, (AccessExpressionMarkings) message.markings, AccessExpressionMarkings.SCHEMA, false); if (message.language != null) { output.writeString(2, message.language, false); @@ -150,8 +143,8 @@ public void mergeFrom(Input input, DefaultTagCloud message) throws IOException { while ((number = input.readFieldNumber(this)) != 0) { switch (number) { case 1: - message.markings = new HashMap(); - input.mergeObject(message.markings, MapSchema.SCHEMA); + message.markings = AccessExpressionMarkings.builder().build(); + input.mergeObject((AccessExpressionMarkings) message.markings, AccessExpressionMarkings.SCHEMA); break; case 2: message.language = input.readString(); diff --git a/web-services/client/src/main/java/datawave/webservice/result/keyword/TagCloudBase.java b/web-services/client/src/main/java/datawave/webservice/result/keyword/TagCloudBase.java index 8840a7ad185..412f0e3a849 100644 --- a/web-services/client/src/main/java/datawave/webservice/result/keyword/TagCloudBase.java +++ b/web-services/client/src/main/java/datawave/webservice/result/keyword/TagCloudBase.java @@ -7,28 +7,23 @@ import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlSeeAlso; -import com.google.common.collect.Maps; - +import datawave.marking.Markings; import datawave.webservice.query.result.event.HasMarkings; import io.protostuff.Message; @XmlAccessorType(XmlAccessType.NONE) @XmlSeeAlso(DefaultTagCloud.class) public abstract class TagCloudBase> implements HasMarkings, Message { - protected transient Map markings; + protected transient Markings markings; protected transient boolean intermediateResult; - public Map getMarkings() { - assureMarkings(); + @Override + public Markings getMarkings() { return markings; } - protected void assureMarkings() { - if (this.markings == null) - this.markings = Maps.newHashMap(); - } - - public void setMarkings(Map markings) { + @Override + public void setMarkings(Markings markings) { this.markings = markings; } diff --git a/web-services/metrics/src/main/java/datawave/query/metrics/ShardTableQueryMetricHandler.java b/web-services/metrics/src/main/java/datawave/query/metrics/ShardTableQueryMetricHandler.java index cb0ea137ae2..88d0a83dd71 100644 --- a/web-services/metrics/src/main/java/datawave/query/metrics/ShardTableQueryMetricHandler.java +++ b/web-services/metrics/src/main/java/datawave/query/metrics/ShardTableQueryMetricHandler.java @@ -75,7 +75,7 @@ import datawave.ingest.mapreduce.job.BulkIngestKey; import datawave.ingest.mapreduce.job.writer.LiveContextWriter; import datawave.ingest.table.config.TableConfigHelper; -import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.microservice.query.Query; import datawave.microservice.query.QueryImpl; import datawave.microservice.query.QueryImpl.Parameter; @@ -279,13 +279,12 @@ private Multimap getEntries(AbstractColumnBasedHandler event.setDataType(type); event.setTimestamp(storedQueryMetric.getCreateDate().getTime()); // get security markings from metric, otherwise default to PUBLIC - Map markings = updatedQueryMetric.getMarkings(); + Markings markings = updatedQueryMetric.getMarkings(); if (markings == null || markings.isEmpty()) { event.setVisibility(new ColumnVisibility(DEFAULT_SECURITY_MARKING)); } else { try { - MarkingFunctions markingFunctions = MarkingFunctions.Factory.createMarkingFunctions(); - event.setVisibility(markingFunctions.translateToColumnVisibility(markings)); + event.setVisibility(markings.toColumnVisibility()); } catch (Exception e) { log.error(e.getMessage(), e); event.setVisibility(new ColumnVisibility(DEFAULT_SECURITY_MARKING)); diff --git a/web-services/query/src/main/java/datawave/webservice/query/factory/Persister.java b/web-services/query/src/main/java/datawave/webservice/query/factory/Persister.java index 6549a0a2c93..61892c38162 100644 --- a/web-services/query/src/main/java/datawave/webservice/query/factory/Persister.java +++ b/web-services/query/src/main/java/datawave/webservice/query/factory/Persister.java @@ -107,7 +107,7 @@ public Query create(String userDN, List dnList, SecurityMarking marking, MultivaluedMap optionalQueryParameters) { Query q = responseObjectFactory.getQueryImpl(); q.initialize(userDN, dnList, queryLogicName, qp, MapUtils.toMultiValueMap(optionalQueryParameters)); - q.setColumnVisibility(marking.toColumnVisibilityString()); + q.setColumnVisibility(marking.toAccessExpressionString()); q.setUncaughtExceptionHandler(new QueryUncaughtExceptionHandler()); Thread.currentThread().setUncaughtExceptionHandler(q.getUncaughtExceptionHandler()); // Persist the query object if required diff --git a/web-services/query/src/main/java/datawave/webservice/query/runner/QueryExecutorBean.java b/web-services/query/src/main/java/datawave/webservice/query/runner/QueryExecutorBean.java index 718f5af012f..c98940db3c3 100644 --- a/web-services/query/src/main/java/datawave/webservice/query/runner/QueryExecutorBean.java +++ b/web-services/query/src/main/java/datawave/webservice/query/runner/QueryExecutorBean.java @@ -616,7 +616,7 @@ private QueryData validateQueryParameters(String queryLogicName, UriInfo uriInfo // These are parameters that aren't passed in by the user, but rather are computed from other sources. PrivateAuditConstants.stripPrivateParameters(queryParameters); queryParameters.add(PrivateAuditConstants.LOGIC_CLASS, queryLogicName); - queryParameters.putSingle(PrivateAuditConstants.COLUMN_VISIBILITY, marking.toColumnVisibilityString()); + queryParameters.putSingle(PrivateAuditConstants.COLUMN_VISIBILITY, marking.toAccessExpressionString()); queryParameters.add(PrivateAuditConstants.USER_DN, qd.userDn); return qd; diff --git a/web-services/query/src/test/java/datawave/webservice/query/logic/TestLegacyBaseQueryLogicTransformer.java b/web-services/query/src/test/java/datawave/webservice/query/logic/TestLegacyBaseQueryLogicTransformer.java index 57bf97e9678..a640dcb7ce9 100644 --- a/web-services/query/src/test/java/datawave/webservice/query/logic/TestLegacyBaseQueryLogicTransformer.java +++ b/web-services/query/src/test/java/datawave/webservice/query/logic/TestLegacyBaseQueryLogicTransformer.java @@ -75,7 +75,7 @@ public void testCreateResponse_ResultsPageComplete() { private static class TestTransformer extends BaseQueryLogicTransformer,EventBase> { BaseQueryResponse response; - public TestTransformer(MarkingFunctions markingFunctions, BaseQueryResponse response) { + public TestTransformer(MarkingFunctions markingFunctions, BaseQueryResponse response) { super(markingFunctions); this.response = response; } diff --git a/web-services/query/src/test/java/datawave/webservice/query/logic/composite/CompositeQueryLogicTest.java b/web-services/query/src/test/java/datawave/webservice/query/logic/composite/CompositeQueryLogicTest.java index 15e8a2597df..5aaa959e554 100644 --- a/web-services/query/src/test/java/datawave/webservice/query/logic/composite/CompositeQueryLogicTest.java +++ b/web-services/query/src/test/java/datawave/webservice/query/logic/composite/CompositeQueryLogicTest.java @@ -41,6 +41,7 @@ import datawave.core.query.logic.composite.CompositeQueryLogic; import datawave.core.query.logic.filtered.FilteredQueryLogic; import datawave.marking.MarkingFunctions; +import datawave.marking.Markings; import datawave.microservice.query.Query; import datawave.microservice.query.QueryImpl; import datawave.security.authorization.AuthorizationException; @@ -152,13 +153,13 @@ public long getTotalResults() { } @Override - public void setMarkings(Map markings) { + public void setMarkings(Markings markings) { this.markings = markings; } @Override - public Map getMarkings() { - return Collections.unmodifiableMap(markings); + public Markings getMarkings() { + return this.markings; } } @@ -184,7 +185,7 @@ public void addResponse(TestQueryResponse response) { public static class TestQueryLogicTransformer extends BaseQueryLogicTransformer,TestQueryResponse> { - public TestQueryLogicTransformer(MarkingFunctions markingFunctions) { + public TestQueryLogicTransformer(MarkingFunctions markingFunctions) { super(markingFunctions); } @@ -224,7 +225,7 @@ public BaseQueryResponse createResponse(List resultList) { public static class DifferentTestQueryLogicTransformer extends BaseQueryLogicTransformer,TestQueryResponse> { - public DifferentTestQueryLogicTransformer(MarkingFunctions markingFunctions) { + public DifferentTestQueryLogicTransformer(MarkingFunctions markingFunctions) { super(markingFunctions); }