diff --git a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveredThing.java b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveredThing.java index 3e2c3e21e95..13048e64f37 100644 --- a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveredThing.java +++ b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveredThing.java @@ -29,6 +29,16 @@ public DiscoveredThing(String term, String field, String type, String date, Stri this.countsByColumnVisibility = countsByColumnVisibility; } + public DiscoveredThing(String term, String columnVisibility) { + this.term = term; + this.field = ""; + this.type = ""; + this.date = ""; + this.columnVisibility = columnVisibility; + this.count = new VLongWritable(0L); + this.countsByColumnVisibility = new MapWritable(); + } + public DiscoveredThing() { count = new VLongWritable(); countsByColumnVisibility = new MapWritable(); diff --git a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveredThingTermIsotope.java b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveredThingTermIsotope.java new file mode 100644 index 00000000000..0239f0e1702 --- /dev/null +++ b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveredThingTermIsotope.java @@ -0,0 +1,40 @@ +package datawave.query.discovery; + +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; +import org.apache.hadoop.io.MapWritable; + +/** + * A variant of DiscoveredThing in which equality if defined as comparing getTerm() only. + */ +public class DiscoveredThingTermIsotope extends DiscoveredThing { + public DiscoveredThingTermIsotope(String term, String field, String type, String date, String columnVisibility, long count, + MapWritable countsByColumnVisibility) { + super(term, field, type, date, columnVisibility, count, countsByColumnVisibility); + + } + + public DiscoveredThingTermIsotope(String term, String columnVisibility) { + super(term, columnVisibility); + } + + public DiscoveredThingTermIsotope() { + super(); + } + + @Override + public boolean equals(Object o) { + if (o instanceof DiscoveredThingTermIsotope) { + DiscoveredThingTermIsotope other = (DiscoveredThingTermIsotope) o; + return new EqualsBuilder().append(getTerm(), other.getTerm()).isEquals(); + } + return false; + } + + @Override + public int hashCode() { + // Ignore super results. + // super.hashCode(); + return new HashCodeBuilder().append(getTerm()).toHashCode(); + } +} diff --git a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveredThingValuesOnlyConditionalTransformer.java b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveredThingValuesOnlyConditionalTransformer.java new file mode 100644 index 00000000000..3c429eab5dc --- /dev/null +++ b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveredThingValuesOnlyConditionalTransformer.java @@ -0,0 +1,25 @@ +package datawave.query.discovery; + +import java.util.function.UnaryOperator; + +/** + * Functional + * + * Converts DiscoveredThing to one containing only the original "Term" and "Column Visibility." Other fields are defined the DiscoveredThing constructor for + * this scenario. The scenario here is a DiscoveredThing is created containing only "Term" and "Column Visibility." + */ +public class DiscoveredThingValuesOnlyConditionalTransformer implements UnaryOperator { + + private final boolean valuesOnly; + + DiscoveredThingValuesOnlyConditionalTransformer(boolean valuesOnly) { + this.valuesOnly = valuesOnly; + } + + public DiscoveredThing apply(DiscoveredThing dt) { + if (valuesOnly) { + return new DiscoveredThing(dt.getTerm(), dt.getColumnVisibility()); + } + return dt; + } +} diff --git a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java index 404d9c29dda..5bf5a922259 100644 --- a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java +++ b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryIterator.java @@ -19,6 +19,8 @@ import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; import org.apache.accumulo.core.security.ColumnVisibility; +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; import org.apache.hadoop.io.ArrayWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.MapWritable; @@ -41,11 +43,14 @@ public class DiscoveryIterator implements SortedKeyValueIterator { private Key key; private Value value; + private Range lastRange; + private Collection columnFamilies; private SortedKeyValueIterator iterator; private boolean separateCountsByColVis = false; private boolean showReferenceCount = false; private boolean reverseIndex = false; private boolean sumCounts = false; + private boolean valuesOnly = false; @Override public DiscoveryIterator deepCopy(IteratorEnvironment env) { @@ -84,15 +89,23 @@ private Multimap getTermsByDatatype() throws IOException { Multimap terms = ArrayListMultimap.create(); Key start = new Key(iterator.getTopKey()); Key key; - // If we should sum up counts, we want to collect the term entries for each date seen for the current field and term of start. Otherwise, we only want - // to collect the term entries for the current field, term, and date of start. + + // If we should sum up counts, we want to collect the term entries for each date seen for the current field and term of start. + // Otherwise, we only want to collect the term entries for the current field, term, and date of start. BiFunction dateMatchingFunction = sumCounts ? (first, second) -> true : this::datesMatch; + // Find all matching entries and parse term entries from them. while (iterator.hasTop() && start.equals((key = iterator.getTopKey()), PartialKey.ROW_COLFAM) && dateMatchingFunction.apply(start, key)) { TermEntry termEntry = new TermEntry(key, iterator.getTopValue()); - if (termEntry.isValid()) + if (termEntry.isValid()) { terms.put(termEntry.getDatatype(), termEntry); - else { + // if 'values only' is selected, then we only need a single TermEntry, the first one. The value of each + // term in TermEntry should be the identical as we iterate in this 'while' loop. Therefore, the first + // one encountered will suffice. + if (valuesOnly) { + return terms; + } + } else { if (log.isTraceEnabled()) { log.trace("Received invalid term entry from key: " + key); } @@ -179,9 +192,10 @@ private DiscoveredThing aggregate(Collection termEntries) { /** * Set the top {@link Key} and {@link Value} of this iterator, created from the given list of {@link DiscoveredThing} instances. */ - private void setTop(List things) { - // We want the key to be the last possible key for this date. Return the key as it is in the index (reversed if necessary) to ensure the keys are - // consistent with the initial seek range. + private void setTop(List things) throws IOException { + + // We want the key to be the last possible key for this date. Return the key as it is in the index (reversed if + // necessary) to ensure the keys are consistent with the initial seek range. DiscoveredThing thing = things.get(0); String row = (this.reverseIndex ? new StringBuilder().append(thing.getTerm()).reverse().toString() : thing.getTerm()); Key newKey = new Key(row, thing.getField(), thing.getDate() + "\uffff"); @@ -189,13 +203,29 @@ private void setTop(List things) { // Create a value from the list of things. ArrayWritable thingArray = new ArrayWritable(DiscoveredThing.class, things.toArray(new DiscoveredThing[0])); Value newValue = new Value(WritableUtils.toByteArray(thingArray)); + if (valuesOnly) { + Key skipKey = new Key(row, "\uffff"); + if (!columnFamilies.isEmpty()) { + skipKey = new Key(row, thing.getField(), "\uffff"); + } + + if (lastRange.contains(skipKey)) { + this.key = skipKey; + Range nextRange = new Range(skipKey, false, lastRange.getEndKey(), lastRange.isEndKeyInclusive()); + this.iterator.seek(nextRange, columnFamilies, false); + } + } else { + this.key = newKey; + + } - this.key = newKey; this.value = newValue; } @Override public void seek(Range range, Collection columnFamilies, boolean inclusive) throws IOException { + this.lastRange = range; + this.columnFamilies = columnFamilies; this.iterator.seek(range, columnFamilies, inclusive); if (log.isTraceEnabled()) { log.trace("My source " + (this.iterator.hasTop() ? "does" : "does not") + " have a top."); @@ -210,6 +240,10 @@ public void init(SortedKeyValueIterator source, Map op this.showReferenceCount = Boolean.parseBoolean(options.get(DiscoveryLogic.SHOW_REFERENCE_COUNT)); this.reverseIndex = Boolean.parseBoolean(options.get(DiscoveryLogic.REVERSE_INDEX)); this.sumCounts = Boolean.parseBoolean(options.get(DiscoveryLogic.SUM_COUNTS)); + this.valuesOnly = Boolean.parseBoolean(options.get(DiscoveryLogic.VALUES_ONLY)); + if (valuesOnly) { + sumCounts = false; + } if (log.isTraceEnabled()) { log.trace("Source: " + source.getClass().getName()); @@ -217,6 +251,7 @@ public void init(SortedKeyValueIterator source, Map op log.trace("Show reference counts only: " + this.showReferenceCount); log.trace("Reverse index: " + this.reverseIndex); log.trace("Sum counts: " + this.sumCounts); + log.trace("Values only: " + this.valuesOnly); } } @@ -325,5 +360,32 @@ public long getUidListSize() { public boolean isValid() { return valid; } + + @Override + public boolean equals(Object o) { + if (o instanceof TermEntry) { + // @formatter:off + TermEntry other = (TermEntry) o; + return new EqualsBuilder().append(getTerm(), other.getTerm()) + .append(getField(), other.getField()) + .append(getVisibility(), other.getVisibility()) + .append(getDate(), other.getDate()) + .append(getDatatype(), other.getDatatype()) + .append(getUidCount(), other.getUidCount()) + .append(getUidListSize(), other.getUidListSize()).isEquals(); + // @formatter:on + } + return false; + } + + @Override + public int hashCode() { + // @formatter:off + return new HashCodeBuilder().append(getTerm()) + .append(getField()).append(getVisibility()) + .append(getDate()).append(getDatatype()) + .append(getUidCount()).append(getUidListSize()).toHashCode(); + // @formatter:on + } } } diff --git a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryLogic.java b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryLogic.java index ed676bef61e..a0eb4316bf3 100644 --- a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryLogic.java +++ b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryLogic.java @@ -27,6 +27,7 @@ import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.Authorizations; +import org.apache.commons.collections4.iterators.UniqueFilterIterator; import org.apache.commons.jexl3.parser.ASTJexlScript; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.math.LongRange; @@ -88,6 +89,11 @@ public class DiscoveryLogic extends ShardIndexQueryTable { */ public static final String SUM_COUNTS = "sum.counts"; + /** + * Used to specify a unique list of values not associated with a field. + */ + public static final String VALUES_ONLY = "values.only"; + /** * Used to specify whether to search against the reversed index. */ @@ -115,6 +121,11 @@ public DiscoveryLogic(DiscoveryLogic other) { this.metadataHelper = other.metadataHelper; } + @Override + public Iterator iterator() { + return getValuesOnly() ? new UniqueFilterIterator<>(this.iterator) : this.iterator; + } + @Override public DiscoveryQueryConfiguration getConfig() { if (this.config == null) { @@ -151,6 +162,9 @@ public GenericQueryConfiguration initialize(AccumuloClient client, Query setting // Check if counts should be summed. setSumCounts(getOrDefaultBoolean(settings, SUM_COUNTS, getSumCounts())); + // Specify values only. Treat associated field, data type, and the like as "don't care." + setValuesOnly(getOrDefaultBoolean(settings, VALUES_ONLY, false)); + // Check if any datatype filters were specified. getConfig().setDatatypeFilter(getOrDefaultSet(settings, QueryParameters.DATATYPE_FILTER_SET, getConfig().getDatatypeFilter())); @@ -580,6 +594,7 @@ private IteratorSetting configureDiscoveryIterator(DiscoveryQueryConfiguration c setting.addOption(SEPARATE_COUNTS_BY_COLVIS, Boolean.toString(config.getSeparateCountsByColVis())); setting.addOption(SHOW_REFERENCE_COUNT, Boolean.toString(config.getShowReferenceCount())); setting.addOption(SUM_COUNTS, Boolean.toString(config.getSumCounts())); + setting.addOption(VALUES_ONLY, Boolean.toString(config.getValuesOnly())); return setting; } @@ -630,14 +645,14 @@ public ShardIndexQueryTable clone() { */ private Iterator transformScanner(final BatchScanner scanner, final QueryData queryData, Set indexedFields) { return concat(transform(scanner.iterator(), new Function,Iterator>() { - DataInputBuffer in = new DataInputBuffer(); + final DataInputBuffer in = new DataInputBuffer(); @Override public Iterator apply(Entry from) { queryData.setLastResult(from.getKey()); Value value = from.getValue(); in.reset(value.get(), value.getSize()); - ArrayWritable aw = new ArrayWritable(DiscoveredThing.class); + ArrayWritable aw = new ArrayWritable(DiscoveredThingTermIsotope.class); try { aw.readFields(in); } catch (IOException e) { @@ -689,4 +704,12 @@ public boolean getSumCounts() { public void setSumCounts(boolean sumCounts) { getConfig().setSumCounts(sumCounts); } + + public void setValuesOnly(boolean valuesOnly) { + getConfig().setValuesOnly(valuesOnly); + } + + public boolean getValuesOnly() { + return getConfig().getValuesOnly(); + } } diff --git a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryQueryConfiguration.java b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryQueryConfiguration.java index 248ae450eb5..49f50b3d22f 100644 --- a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryQueryConfiguration.java +++ b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryQueryConfiguration.java @@ -22,6 +22,7 @@ public class DiscoveryQueryConfiguration extends ShardIndexQueryConfiguration im private boolean separateCountsByColVis = false; private boolean showReferenceCount = false; private boolean sumCounts = false; + private boolean valuesOnly = false; public DiscoveryQueryConfiguration() {} @@ -131,6 +132,10 @@ public boolean getSumCounts() { return sumCounts; } + public boolean getValuesOnly() { + return valuesOnly; + } + public void setSeparateCountsByColVis(boolean separateCountsByColVis) { this.separateCountsByColVis = separateCountsByColVis; } @@ -144,6 +149,10 @@ public void setSumCounts(boolean sumCounts) { this.sumCounts = sumCounts; } + public void setValuesOnly(boolean valuesOnly) { + this.valuesOnly = valuesOnly; + } + @Override public DiscoveryQueryConfiguration checkpoint() { // Create a new config that only contains what is needed to execute the specified ranges @@ -172,6 +181,6 @@ public int hashCode() { public String toString() { return new StringJoiner(", ", DiscoveryQueryConfiguration.class.getSimpleName() + "[", "]").add("literals=" + literals).add("patterns=" + patterns) .add("ranges=" + ranges).add("separateCountsByColVis=" + separateCountsByColVis).add("showReferenceCount=" + showReferenceCount) - .add("sumCounts=" + sumCounts).toString(); + .add("sumCounts=" + sumCounts).add("valuesOnly=" + valuesOnly).toString(); } } diff --git a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryTransformer.java index bbabc7c9537..dc30c85e8ee 100644 --- a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryTransformer.java @@ -1,11 +1,17 @@ package datawave.query.discovery; +import static datawave.query.discovery.DiscoveryLogic.VALUES_ONLY; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.function.BiFunction; +import java.util.function.Function; import org.apache.accumulo.core.security.ColumnVisibility; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.io.Writable; import com.google.common.base.Preconditions; @@ -16,9 +22,9 @@ import datawave.marking.MarkingFunctions; import datawave.marking.MarkingFunctions.Exception; import datawave.microservice.query.Query; +import datawave.microservice.query.QueryImpl; import datawave.query.model.QueryModel; import datawave.webservice.query.cachedresults.CacheableQueryRow; -import datawave.webservice.query.exception.QueryException; import datawave.webservice.query.result.event.EventBase; import datawave.webservice.query.result.event.FieldBase; import datawave.webservice.query.result.event.Metadata; @@ -26,12 +32,14 @@ import datawave.webservice.result.BaseQueryResponse; import datawave.webservice.result.EventQueryResponseBase; +@SuppressWarnings({"rawtypes"}) public class DiscoveryTransformer extends BaseQueryLogicTransformer implements CacheableLogic { private List variableFieldList = null; - private BaseQueryLogic logic = null; - private QueryModel myQueryModel = null; + private final BaseQueryLogic logic; + private QueryModel myQueryModel; private MarkingFunctions markingFunctions; - private ResponseObjectFactory responseObjectFactory; + private final ResponseObjectFactory responseObjectFactory; + private boolean valuesOnly = false; public DiscoveryTransformer(BaseQueryLogic logic, Query settings, QueryModel qm) { super(new MarkingFunctions.Default()); @@ -39,46 +47,80 @@ public DiscoveryTransformer(BaseQueryLogic logic, Query setting this.responseObjectFactory = logic.getResponseObjectFactory(); this.logic = logic; this.myQueryModel = qm; + this.valuesOnly = getOrDefaultBoolean(settings, VALUES_ONLY, false); } - @Override - public EventBase transform(DiscoveredThing thing) { - Preconditions.checkNotNull(thing, "Received a null object to transform!"); + /** + * Variant of a field list that contains only a value field. + */ + BiFunction,List> generateValuesOnlyFieldList = (x, y) -> { + List fields = new ArrayList<>(); - EventBase event = this.responseObjectFactory.getEvent(); - Map markings; - try { - markings = this.markingFunctions.translateFromColumnVisibility(new ColumnVisibility(thing.getColumnVisibility())); - } catch (Exception e) { - throw new RuntimeException("could not parse to markings: " + thing.getColumnVisibility()); - } - event.setMarkings(markings); + fields.add(this.makeField("VALUE", y, "", 0L, x.getTerm())); + return fields; + }; + + /** + * Variant of field list that contains a standard, default set of fields. + */ + BiFunction,List> generateFieldList = (x, y) -> { List fields = new ArrayList<>(); - fields.add(this.makeField("VALUE", markings, "", 0L, thing.getTerm())); - /** - * Added query model to alias FIELD + fields.add(this.makeField("VALUE", y, "", 0L, x.getTerm())); + /* + * Added query model to alias FIELD, if DiscoveredThing::field both not NULL and not empty. */ - fields.add(this.makeField("FIELD", markings, "", 0L, myQueryModel.aliasFieldNameReverseModel(thing.getField()))); - fields.add(this.makeField("DATE", markings, "", 0L, thing.getDate())); - fields.add(this.makeField("DATA TYPE", markings, "", 0L, thing.getType())); + Optional fieldOFThing = Optional.ofNullable(x.getField()); + fieldOFThing.filter(i -> !i.isBlank()).ifPresent(i -> fields.add(this.makeField("FIELD", y, "", 0L, myQueryModel.aliasFieldNameReverseModel(i)))); + + fields.add(this.makeField("DATE", y, "", 0L, x.getDate())); + fields.add(this.makeField("DATA TYPE", y, "", 0L, x.getType())); // If requested return counts separated by colvis, all counts by colvis could be > total record count - if (thing.getCountsByColumnVisibility() != null && !thing.getCountsByColumnVisibility().isEmpty()) { - for (Map.Entry entry : thing.getCountsByColumnVisibility().entrySet()) { + if (x.getCountsByColumnVisibility() != null && !x.getCountsByColumnVisibility().isEmpty()) { + for (Map.Entry entry : x.getCountsByColumnVisibility().entrySet()) { try { - Map eMarkings = this.markingFunctions.translateFromColumnVisibility(new ColumnVisibility(entry.getKey().toString())); + this.markingFunctions.translateFromColumnVisibility(new ColumnVisibility(entry.getKey().toString())); fields.add(this.makeField("RECORD COUNT", new HashMap<>(), entry.getKey().toString(), 0L, entry.getValue().toString())); } catch (Exception e) { - throw new RuntimeException("could not parse to markings: " + thing.getColumnVisibility()); + throw new RuntimeException("could not parse to markings: " + x.getColumnVisibility()); } } } else { - fields.add(this.makeField("RECORD COUNT", markings, "", 0L, Long.toString(thing.getCount()))); + fields.add(this.makeField("RECORD COUNT", y, "", 0L, Long.toString(x.getCount()))); } + return fields; + }; + + final Map,List>> mapMapGenerator = new HashMap<>(); + { + mapMapGenerator.put("VALUES_ONLY", generateValuesOnlyFieldList); + mapMapGenerator.put("STANDARD", generateFieldList); + } + + /** + * Factory to get a particular field list generator. Different scenarios may call for different field lists. + * + * @param isValuesOnly + * @return + */ + BiFunction,List> getMapGenerator(boolean isValuesOnly) { + return mapMapGenerator.get(isValuesOnly ? "VALUES_ONLY" : "STANDARD"); + } + @SuppressWarnings({"rawtypes", "unchecked"}) + @Override + public EventBase transform(DiscoveredThing thing) { + Preconditions.checkNotNull(thing, "Received a null object to transform!"); + + EventBase event = this.responseObjectFactory.getEvent(); + + Map markings = markingsFromVisibility.apply(thing.getColumnVisibility()); + event.setMarkings(markings); + + List fields = getMapGenerator(valuesOnly).apply(thing, markings); event.setFields(fields); event.setSizeInBytes(fields.size() * 6L); @@ -92,8 +134,9 @@ public EventBase transform(DiscoveredThing thing) { return event; } - protected FieldBase makeField(String name, Map markings, String columnVisibility, Long timestamp, Object value) { - FieldBase field = this.responseObjectFactory.getField(); + @SuppressWarnings({"rawtypes"}) + protected FieldBase makeField(String name, Map markings, String columnVisibility, Long timestamp, Object value) { + FieldBase field = this.responseObjectFactory.getField(); field.setName(name); field.setMarkings(markings); field.setColumnVisibility(columnVisibility); @@ -102,6 +145,7 @@ protected FieldBase makeField(String name, Map markings, Strin return field; } + @SuppressWarnings({"rawtypes"}) @Override public BaseQueryResponse createResponse(List resultList) { EventQueryResponseBase response = this.responseObjectFactory.getEventQueryResponse(); @@ -116,8 +160,9 @@ public BaseQueryResponse createResponse(List resultList) { return response; } + @SuppressWarnings({"unchecked"}) @Override - public CacheableQueryRow writeToCache(Object o) throws QueryException { + public CacheableQueryRow writeToCache(Object o) { EventBase event = (EventBase) o; CacheableQueryRow cqo = responseObjectFactory.getCacheableQueryRow(); @@ -135,6 +180,7 @@ public CacheableQueryRow writeToCache(Object o) throws QueryException { return cqo; } + @SuppressWarnings({"unchecked"}) @Override public Object readFromCache(CacheableQueryRow cacheableQueryRow) { if (this.variableFieldList == null) { @@ -175,4 +221,29 @@ public Object readFromCache(CacheableQueryRow cacheableQueryRow) { event.setFields(fieldList); return event; } + + Function> markingsFromVisibility = x -> { + try { + return this.markingFunctions.translateFromColumnVisibility(new ColumnVisibility(x)); + } catch (Exception e) { + throw new RuntimeException("could not parse to markings: " + x); + } + }; + + /** + * If present, return the value of the given parameter from the given settings as a boolean, or return the default value otherwise. + */ + private boolean getOrDefaultBoolean(Query settings, String parameterName, boolean defaultValue) { + String value = getTrimmedParameter(settings, parameterName); + return StringUtils.isBlank(value) ? defaultValue : Boolean.parseBoolean(value); + } + + /** + * Return the trimmed value of the given parameter from the given settings, or null if a value is not present. + */ + private String getTrimmedParameter(Query settings, String parameterName) { + QueryImpl.Parameter parameter = settings.findParameter(parameterName); + return parameter != null ? parameter.getParameterValue().trim() : null; + } + } diff --git a/warehouse/query-core/src/test/java/datawave/query/discovery/DiscoveredThingTest.java b/warehouse/query-core/src/test/java/datawave/query/discovery/DiscoveredThingTest.java new file mode 100644 index 00000000000..38a3cd4c445 --- /dev/null +++ b/warehouse/query-core/src/test/java/datawave/query/discovery/DiscoveredThingTest.java @@ -0,0 +1,44 @@ +package datawave.query.discovery; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; + +import org.apache.hadoop.io.MapWritable; +import org.junit.jupiter.api.Test; + +public class DiscoveredThingTest { + @Test + public void testDiscoveredThingSimpleEqualityTest() { + DiscoveredThing thing1 = new DiscoveredThing("bbc", "NETWORK", "csv", "20130101", "FOO", 240L, new MapWritable()); + DiscoveredThing thing2 = new DiscoveredThing("bbc", "NETWORK", "csv", "20130101", "FOO", 240L, new MapWritable()); + assertEquals(thing1, thing2); + + DiscoveredThing thing3 = new DiscoveredThing("", "", "", "", "", 0L, new MapWritable()); + DiscoveredThing thing4 = new DiscoveredThing("", "", "", "", "", 0L, new MapWritable()); + assertEquals(thing3, thing4); + + DiscoveredThing thing5 = new DiscoveredThing("", "", "", "", "", 0L, null); + DiscoveredThing thing6 = new DiscoveredThing("", "", "", "", "", 0L, null); + assertEquals(thing5, thing6); + + } + + @Test + public void testDiscoveredThingSimpleInequalityTest() { + DiscoveredThing thing1 = new DiscoveredThing("bbc", "NETWORK", "csv", "20130101", "FOO", 240L, new MapWritable()); + DiscoveredThing thing2 = new DiscoveredThing("bbc", "NETWORK", "csv", "20130102", "FOO", 240L, new MapWritable()); + assertNotEquals(thing1, thing2); + + DiscoveredThing thing3 = new DiscoveredThing("", "wanda", "", "", "", 0L, new MapWritable()); + DiscoveredThing thing4 = new DiscoveredThing("", "panda", "", "", "", 0L, new MapWritable()); + assertNotEquals(thing3, thing4); + + DiscoveredThing thing5 = new DiscoveredThing("", "wands", "", "", "", 0L, null); + DiscoveredThing thing6 = new DiscoveredThing("", "wands", "", "", "", 0L, new MapWritable()); + assertNotEquals(thing5, thing6); + + DiscoveredThing thing7 = new DiscoveredThing("", "wanda", "", "", "", 0L, null); + DiscoveredThing thing8 = new DiscoveredThing("", "panda", "", "", "", 0L, null); + assertNotEquals(thing7, thing8); + } +} diff --git a/warehouse/query-core/src/test/java/datawave/query/discovery/DiscoveryLogicTest.java b/warehouse/query-core/src/test/java/datawave/query/discovery/DiscoveryLogicTest.java index f0512fc2ac6..81ccd7957e9 100644 --- a/warehouse/query-core/src/test/java/datawave/query/discovery/DiscoveryLogicTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/discovery/DiscoveryLogicTest.java @@ -1,5 +1,8 @@ package datawave.query.discovery; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; + import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collections; @@ -26,11 +29,10 @@ import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.hadoop.io.MapWritable; import org.apache.log4j.Logger; -import org.assertj.core.api.Assertions; -import org.junit.After; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import datawave.core.query.configuration.GenericQueryConfiguration; import datawave.core.query.result.event.DefaultResponseObjectFactory; @@ -58,16 +60,16 @@ public class DiscoveryLogicTest { private String query; private String startDate; private String endDate; - private Map parameters = new HashMap<>(); + private final Map parameters = new HashMap<>(); private final List expected = new ArrayList<>(); - @BeforeClass + @BeforeAll public static void setUp() { System.setProperty(MetadataHelperFactory.ALL_AUTHS_PROPERTY, QUERY_AUTHS); } - @Before + @BeforeEach public void setup() throws Throwable { initClient(); writeData(); @@ -94,7 +96,7 @@ private void writeData() throws Throwable { writeEntries("ROOSTER", "onyx", "csv", "BAR", "20130102", 5, 24, 2); writeEntries("ROOSTER", "onyx", "csv", "BAR", "20130103", 5, 24, 20); writeEntries("NETWORK", "bbc", "csv", "FOO", "20130101", 10, 24, 20); - writeEntries("NETWORK", "bbc", "csv", "FOO", "20130102", 10, 24, 20); + writeEntries("NETWORK", "bbc", "csv", "BAR", "20130102", 10, 24, 20); // formerly FOO writeEntries("NETWORK", "bbc", "csv", "FOO", "20130103", 10, 24, 20); writeEntries("OCCUPATION", "skydiver", "text", "FOO", "20130101", 10, 10, 5); writeEntries("OCCUPATION", "skydiver", "text", "FOO", "20130102", 10, 10, 5); @@ -218,7 +220,7 @@ private void initLogic() { logic.setMetadataHelperFactory(new MetadataHelperFactory()); } - @After + @AfterEach public void tearDown() throws Exception { query = null; startDate = null; @@ -241,16 +243,31 @@ private void assertQueryResults() throws Exception { logic.setupQuery(config); Iterator iterator = logic.iterator(); List actual = new ArrayList<>(); + + DiscoveredThingValuesOnlyConditionalTransformer dtvoct = new DiscoveredThingValuesOnlyConditionalTransformer(logic.getValuesOnly()); while (iterator.hasNext()) { - actual.add(iterator.next()); + DiscoveredThing dtee = iterator.next(); + actual.add(dtvoct.apply(dtee)); } - Assertions.assertThat(actual).hasSize(expected.size()); + assertEquals(expected.size(), actual.size()); for (int i = 0; i < expected.size(); i++) { DiscoveredThing actualThing = actual.get(i); DiscoveredThing expectedThing = expected.get(i); - Assertions.assertThat(actualThing).isEqualTo(expectedThing); - Assertions.assertThat(actualThing.getCountsByColumnVisibility()).isEqualTo(expectedThing.getCountsByColumnVisibility()); + assertInstanceOf(DiscoveredThing.class, actualThing); + // N.B.: DiscoveredThingTermIsotope extends DiscoveredThing. Convert to common format for comparison. + //@formatter:off + DiscoveredThing actualThingg = new DiscoveredThing(actualThing.getTerm(), + actualThing.getField(), + actualThing.getType(), + actualThing.getDate(), + actualThing.getColumnVisibility(), + actualThing.getCount(), + actualThing.getCountsByColumnVisibility()); + //@formatter:on + assertEquals(expectedThing, actualThingg); + + assertEquals(expectedThing.getCountsByColumnVisibility(), actualThing.getCountsByColumnVisibility()); } } @@ -281,7 +298,7 @@ public void testLiterals() throws Exception { givenEndDate("20130102"); expect(new DiscoveredThing("bbc", "NETWORK", "csv", "20130101", "FOO", 240L, new MapWritable())); - expect(new DiscoveredThing("bbc", "NETWORK", "csv", "20130102", "FOO", 240L, new MapWritable())); + expect(new DiscoveredThing("bbc", "NETWORK", "csv", "20130102", "BAR", 240L, new MapWritable())); expect(new DiscoveredThing("onyx", "POKEMON", "csv", "20130101", "FOO", 100L, new MapWritable())); expect(new DiscoveredThing("onyx", "POKEMON", "csv", "20130102", "FOO", 10L, new MapWritable())); expect(new DiscoveredThing("onyx", "ROCK", "csv", "20130101", "FOO", 1L, new MapWritable())); @@ -299,7 +316,7 @@ public void testPatterns() throws Exception { givenEndDate("20130102"); expect(new DiscoveredThing("bbc", "NETWORK", "csv", "20130101", "FOO", 240L, new MapWritable())); - expect(new DiscoveredThing("bbc", "NETWORK", "csv", "20130102", "FOO", 240L, new MapWritable())); + expect(new DiscoveredThing("bbc", "NETWORK", "csv", "20130102", "BAR", 240L, new MapWritable())); expect(new DiscoveredThing("onyx", "POKEMON", "csv", "20130101", "FOO", 100L, new MapWritable())); expect(new DiscoveredThing("onyx", "POKEMON", "csv", "20130102", "FOO", 10L, new MapWritable())); expect(new DiscoveredThing("onyx", "ROCK", "csv", "20130101", "FOO", 1L, new MapWritable())); @@ -427,7 +444,7 @@ public void testSumCountsForLiterals() throws Exception { givenEndDate("20130102"); givenParameter(DiscoveryLogic.SUM_COUNTS, "true"); - expect(new DiscoveredThing("bbc", "NETWORK", "csv", "", "FOO", 480L, new MapWritable())); + expect(new DiscoveredThing("bbc", "NETWORK", "csv", "", "BAR&FOO", 480L, new MapWritable())); expect(new DiscoveredThing("onyx", "POKEMON", "csv", "", "FOO", 110L, new MapWritable())); expect(new DiscoveredThing("onyx", "ROCK", "csv", "", "FOO", 4L, new MapWritable())); expect(new DiscoveredThing("onyx", "ROOSTER", "csv", "", "BAR", 240L, new MapWritable())); @@ -442,7 +459,7 @@ public void testSumCountsForPatterns() throws Exception { givenEndDate("20130102"); givenParameter(DiscoveryLogic.SUM_COUNTS, "true"); - expect(new DiscoveredThing("bbc", "NETWORK", "csv", "", "FOO", 480L, new MapWritable())); + expect(new DiscoveredThing("bbc", "NETWORK", "csv", "", "BAR&FOO", 480L, new MapWritable())); expect(new DiscoveredThing("onyx", "POKEMON", "csv", "", "FOO", 110L, new MapWritable())); expect(new DiscoveredThing("onyx", "ROCK", "csv", "", "FOO", 4L, new MapWritable())); expect(new DiscoveredThing("onyx", "ROOSTER", "csv", "", "BAR", 240L, new MapWritable())); @@ -520,4 +537,81 @@ public void testSumCountsForReverse() throws Exception { assertQueryResults(); } + + @Test + public void testValuesOnlyForLiterals() throws Exception { + givenQuery("bbc OR onyx"); + givenStartDate("20130101"); + givenEndDate("20130102"); + givenParameter(DiscoveryLogic.SUM_COUNTS, "true"); + givenParameter(DiscoveryLogic.VALUES_ONLY, "true"); + + expect(new DiscoveredThing("bbc", "", "", "", "FOO", 0L, new MapWritable())); + expect(new DiscoveredThing("onyx", "", "", "", "FOO", 0L, new MapWritable())); + assertQueryResults(); + } + + @Test + public void testValuesOnlyForLiteralsFalseSumCount() throws Exception { + givenQuery("bbc OR onyx"); + givenStartDate("20130101"); + givenEndDate("20130102"); + givenParameter(DiscoveryLogic.SUM_COUNTS, "false"); + givenParameter(DiscoveryLogic.VALUES_ONLY, "true"); + + expect(new DiscoveredThing("bbc", "", "", "", "FOO", 0L, new MapWritable())); + expect(new DiscoveredThing("onyx", "", "", "", "FOO", 0L, new MapWritable())); + assertQueryResults(); + } + + @Test + public void testValuesOnlyForPatterns() throws Exception { + givenQuery("*yx OR b*"); + givenStartDate("20130101"); + givenEndDate("20130102"); + givenParameter(DiscoveryLogic.SUM_COUNTS, "true"); + givenParameter(DiscoveryLogic.VALUES_ONLY, "true"); + + expect(new DiscoveredThing("bbc", "", "", "", "FOO", 0L, new MapWritable())); + expect(new DiscoveredThing("onyx", "", "", "", "FOO", 0L, new MapWritable())); + + assertQueryResults(); + } + + @Test + public void testValuesOnlyForPatternsNotFound() throws Exception { + givenQuery("*nixon OR ford*"); + givenStartDate("20130101"); + givenEndDate("20130102"); + givenParameter(DiscoveryLogic.SUM_COUNTS, "true"); + givenParameter(DiscoveryLogic.VALUES_ONLY, "true"); + + assertQueryResults(); + } + + @Test + public void testValuesOnlyForFieldedLiterals() throws Exception { + givenQuery("rock:onyx OR pokemon:onyx"); + givenStartDate("20130101"); + givenEndDate("20130102"); + + givenParameter(DiscoveryLogic.SUM_COUNTS, "true"); + givenParameter(DiscoveryLogic.VALUES_ONLY, "true"); + + expect(new DiscoveredThing("onyx", "", "", "", "FOO", 0L, new MapWritable())); + assertQueryResults(); + } + + @Test + public void testValuesOnlyForFieldedLiteralsExtendedRange() throws Exception { + givenQuery("rock:onyx OR pokemon:onyx"); + givenStartDate("20130101"); + givenEndDate("20130104"); + + givenParameter(DiscoveryLogic.SUM_COUNTS, "true"); + givenParameter(DiscoveryLogic.VALUES_ONLY, "true"); + + expect(new DiscoveredThing("onyx", "", "", "", "FOO", 0L, new MapWritable())); + assertQueryResults(); + } } diff --git a/warehouse/query-core/src/test/java/datawave/query/discovery/DiscoveryTransformerTest.java b/warehouse/query-core/src/test/java/datawave/query/discovery/DiscoveryTransformerTest.java new file mode 100644 index 00000000000..f989a7d0aeb --- /dev/null +++ b/warehouse/query-core/src/test/java/datawave/query/discovery/DiscoveryTransformerTest.java @@ -0,0 +1,301 @@ +package datawave.query.discovery; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.ByteArrayOutputStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBElement; +import javax.xml.bind.JAXBException; +import javax.xml.bind.Marshaller; +import javax.xml.namespace.QName; + +import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.core.security.ColumnVisibility; +import org.apache.hadoop.io.MapWritable; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import datawave.accumulo.inmemory.InMemoryAccumuloClient; +import datawave.accumulo.inmemory.InMemoryInstance; +import datawave.core.query.result.event.DefaultResponseObjectFactory; +import datawave.marking.MarkingFunctions; +import datawave.microservice.query.Query; +import datawave.microservice.query.QueryImpl; +import datawave.query.QueryParameters; +import datawave.query.QueryTestTableHelper; +import datawave.query.composite.CompositeMetadataHelper; +import datawave.query.model.QueryModel; +import datawave.query.util.AllFieldMetadataHelper; +import datawave.query.util.MetadataHelper; +import datawave.query.util.MetadataHelperFactory; +import datawave.query.util.TypeMetadataHelper; +import datawave.util.TableName; +import datawave.webservice.query.result.event.DefaultEvent; +import datawave.webservice.query.result.event.DefaultField; +import datawave.webservice.query.result.event.EventBase; +import datawave.webservice.query.result.event.FieldBase; +import datawave.webservice.query.result.event.Metadata; + +public class DiscoveryTransformerTest { + private static final String METADATA_TABLE_NAME = "DatawaveMetadata"; + private MetadataHelper helper; + private AllFieldMetadataHelper allFieldHelper; + private final String[] authorizations = {"FOO", "BAR"}; + private static AccumuloClient client; + private DiscoveryLogic logic; + + @BeforeAll + public static void beforeAll() throws Exception { + InMemoryInstance instance = new InMemoryInstance(DiscoveryTransformerTest.class.getName()); + client = new InMemoryAccumuloClient("", instance); + client.tableOperations().create(METADATA_TABLE_NAME); + } + + @BeforeEach + public void setup() throws Throwable { + initLogic(); + } + + private void initLogic() { + logic = new DiscoveryLogic(); + logic.setIndexTableName(TableName.SHARD_INDEX); + logic.setReverseIndexTableName(TableName.SHARD_RINDEX); + logic.setModelTableName(QueryTestTableHelper.METADATA_TABLE_NAME); + logic.setMetadataTableName(QueryTestTableHelper.METADATA_TABLE_NAME); + logic.setModelName("DATAWAVE"); + logic.setFullTableScanEnabled(false); + logic.setMaxResults(-1); + logic.setMaxWork(-1); + logic.setAllowLeadingWildcard(true); + logic.setResponseObjectFactory(new DefaultResponseObjectFactory()); + logic.setMarkingFunctions(new MarkingFunctions.Default()); + logic.setMetadataHelperFactory(new MetadataHelperFactory()); + } + + @BeforeEach + public void beforeEach() { + allFieldHelper = createAllFieldMetadataHelper(); + helper = createMetadataHelper(allFieldHelper); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + @Test + public void testTermsOnlyDiscoveredThing() throws TableNotFoundException { + + // Create settings. + Query settings = new QueryImpl(); + settings.setQuery("event:20241218_0/samplecsv/1.2.3"); + settings.addParameter(QueryParameters.DECODE_VIEW, "true"); + settings.setQueryAuthorizations("A"); + Map parameters = new HashMap<>(); + parameters.put(DiscoveryLogic.VALUES_ONLY, "true"); + settings.addParameters(parameters); + + // Create query model. + QueryModel qm = helper.getQueryModel(METADATA_TABLE_NAME, "TEST_MODEL"); + + // Create transformer. + DiscoveryTransformer transformer = new DiscoveryTransformer(logic, settings, qm); + + DiscoveredThing thing = new DiscoveredThing("onyx", "", "", "", "FOO", 0L, new MapWritable()); + + EventBase expectedEventBase = new DefaultEvent(); + expectedEventBase.setMetadata(discoveredThingToMetadata(thing)); + Map markings = markingsFromVisibility.apply(thing.getColumnVisibility()); + + expectedEventBase.setMarkings(markings); + expectedEventBase.setFields(discoveredThingToFieldValuesOnly(thing, markings)); + expectedEventBase.setSizeInBytes(6L); + + // This is the heart of the test. + EventBase eb = transformer.transform(thing); + + String serializeExpectedEvent = serializeEvent.apply(expectedEventBase); + String serializedEvent = serializeEvent.apply(eb); + assertEquals(serializeExpectedEvent, serializedEvent); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + @Test + public void testTermsOnlySingleDiscoveredThingLoaded() throws TableNotFoundException { + // Create settings. + Query settings = new QueryImpl(); + settings.setQuery("event:20241218_0/samplecsv/1.2.3"); + settings.addParameter(QueryParameters.DECODE_VIEW, "true"); + settings.setQueryAuthorizations("A"); + Map parameters = new HashMap<>(); + parameters.put(DiscoveryLogic.VALUES_ONLY, "true"); + settings.addParameters(parameters); + + // Create query model. + QueryModel qm = helper.getQueryModel(METADATA_TABLE_NAME, "TEST_MODEL"); + + // Create transformer. + DiscoveryTransformer transformer = new DiscoveryTransformer(logic, settings, qm); + + DiscoveredThing thing = new DiscoveredThing("bbc", "NETWORK", "csv", "20130101", "FOO", 240L, new MapWritable()); + + EventBase expectedEventBase = new DefaultEvent(); + expectedEventBase.setMetadata(discoveredThingToMetadata(thing)); + Map markings = markingsFromVisibility.apply(thing.getColumnVisibility()); + + expectedEventBase.setMarkings(markings); + expectedEventBase.setFields(discoveredThingToFieldValuesOnly(thing, markings)); + expectedEventBase.setSizeInBytes(6L); + + // This is the heart of the test. + EventBase eb = transformer.transform(thing); + + String serializeExpectedEvent = serializeEvent.apply(expectedEventBase); + String serializedEvent = serializeEvent.apply(eb); + assertEquals(serializeExpectedEvent, serializedEvent); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + @Test + public void testSingleDiscoveredThing() throws TableNotFoundException { + // Create settings. + Query settings = new QueryImpl(); + settings.setQuery("event:20241218_0/samplecsv/1.2.3"); + settings.addParameter(QueryParameters.DECODE_VIEW, "true"); + settings.setQueryAuthorizations("A"); + Map parameters = new HashMap<>(); + parameters.put(DiscoveryLogic.VALUES_ONLY, "false"); + settings.addParameters(parameters); + + // Create query model. + QueryModel qm = helper.getQueryModel(METADATA_TABLE_NAME, "TEST_MODEL"); + + // Create transformer. + DiscoveryTransformer transformer = new DiscoveryTransformer(logic, settings, qm); + + DiscoveredThing thing = new DiscoveredThing("bbc", "NETWORK", "csv", "20130101", "FOO", 240L, new MapWritable()); + + EventBase expectedEventBase = new DefaultEvent(); + expectedEventBase.setMetadata(discoveredThingToMetadata(thing)); + Map markings = markingsFromVisibility.apply(thing.getColumnVisibility()); + + expectedEventBase.setMarkings(markings); + expectedEventBase.setFields(discoveredThingToField(thing, markings)); + expectedEventBase.setSizeInBytes(6L); + + // This is the heart of the test. + EventBase eb = transformer.transform(thing); + + String serializeExpectedEvent = serializeEvent.apply(expectedEventBase); + String serializedEvent = serializeEvent.apply(eb); + assertEquals(serializeExpectedEvent, serializedEvent); + } + + @SuppressWarnings("rawtypes") + private final Function serializeEvent = (x) -> { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + try { + JAXBContext jaxbContext = JAXBContext.newInstance(EventBase.class); + Marshaller marshaller = jaxbContext.createMarshaller(); + + // Construct a root element in order to unmarshall. + JAXBElement root = new JAXBElement<>(new QName("", "event"), EventBase.class, x); + marshaller.marshal(root, bos); + } catch (JAXBException e) { + throw new RuntimeException(e); + } + + return bos.toString(); + }; + + private final Function> markingsFromVisibility = x -> { + try { + return this.logic.getMarkingFunctions().translateFromColumnVisibility(new ColumnVisibility(x)); + } catch (MarkingFunctions.Exception e) { + throw new RuntimeException("could not parse to markings: " + x); + } + }; + + @SuppressWarnings("rawtypes") + private List discoveredThingToFieldValuesOnly(DiscoveredThing thing, Map markings) { + List fields = new ArrayList<>(); + + if (Optional.ofNullable(thing.getTerm()).isPresent()) { + fields.add(makeField("VALUE", markings, "", 0L, thing.getTerm())); + } + return fields; + } + + @SuppressWarnings("rawtypes") + private List discoveredThingToField(DiscoveredThing thing, Map markings) { + List fields = new ArrayList<>(); + + if (Optional.ofNullable(thing.getTerm()).isPresent()) { + fields.add(makeField("VALUE", markings, "", 0L, thing.getTerm())); + } + + if (Optional.ofNullable(thing.getField()).isPresent()) { + fields.add(makeField("FIELD", markings, "", 0L, thing.getField())); + } + + if (Optional.ofNullable(thing.getDate()).isPresent()) { + fields.add(makeField("DATE", markings, "", 0L, thing.getDate())); + } + + if (Optional.ofNullable(thing.getType()).isPresent()) { + fields.add(makeField("DATA TYPE", markings, "", 0L, thing.getType())); + } + + if (thing.getCount() > 0L) { + fields.add(makeField("RECORD COUNT", markings, "", 0L, String.valueOf(thing.getCount()))); + } + + return fields; + } + + private AllFieldMetadataHelper createAllFieldMetadataHelper() { + final Set allAuths = Collections.singleton(new Authorizations(authorizations)); + final Set auths = Collections.singleton(new Authorizations(authorizations)); + TypeMetadataHelper typeMetadataHelper = new TypeMetadataHelper(new HashMap<>(), auths, client, METADATA_TABLE_NAME, auths, false); + CompositeMetadataHelper compositeMetadataHelper = new CompositeMetadataHelper(client, METADATA_TABLE_NAME, auths); + return new AllFieldMetadataHelper(typeMetadataHelper, compositeMetadataHelper, client, METADATA_TABLE_NAME, auths, allAuths); + } + + @SuppressWarnings("rawtypes") + private FieldBase makeField(String name, Map markings, String columnVisibility, Long timestamp, Object value) { + FieldBase field = new DefaultField(); + field.setName(name); + field.setMarkings(markings); + field.setColumnVisibility(columnVisibility); + field.setTimestamp(timestamp); + field.setValue(value); + return field; + } + + private Metadata discoveredThingToMetadata(DiscoveredThing thing) { + Metadata metadata = new Metadata(); + metadata.setInternalId(""); + metadata.setDataType(thing.getType()); + metadata.setRow(thing.getTerm()); + metadata.setTable(logic.getTableName()); + return metadata; + } + + private MetadataHelper createMetadataHelper(AllFieldMetadataHelper allFieldHelper) { + if (allFieldHelper == null) { + allFieldHelper = createAllFieldMetadataHelper(); + } + + Set userAuths = Collections.singleton(new Authorizations(authorizations)); + Set metadataAuths = Collections.singleton(new Authorizations(authorizations)); + return new MetadataHelper(allFieldHelper, metadataAuths, client, METADATA_TABLE_NAME, userAuths, metadataAuths); + } +}