From 92b0018cf262aa0e31f64bfdfd89c85d3def4025 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 15 Apr 2026 11:04:00 -0700 Subject: [PATCH 1/6] update metaChecksum algorithm digest a single 0 byte after each list item PrimitiveWrapper by itself and inside collections handle arrays of primitives, usually from PrimtiveWrapper unwrap use cases: CAOM-2.5 algorithm updates --- cadc-util/build.gradle | 2 +- .../java/org/opencadc/persist/Entity.java | 144 ++++++++++-------- .../java/ca/nrc/cadc/date/DateUtilTest.java | 10 ++ .../java/org/opencadc/persist/EntityTest.java | 55 ++++--- .../opencadc/persist/EntityVisitorTest.java | 2 +- .../org/opencadc/persist/SampleEntity.java | 10 +- .../org/opencadc/persist/SampleEntityV2.java | 4 +- 7 files changed, 135 insertions(+), 92 deletions(-) diff --git a/cadc-util/build.gradle b/cadc-util/build.gradle index 0a9f8c87..564392ae 100644 --- a/cadc-util/build.gradle +++ b/cadc-util/build.gradle @@ -15,7 +15,7 @@ sourceCompatibility = 1.8 group = 'org.opencadc' -version = '1.12.15' +version = '1.12.16' description = 'OpenCADC core utility library' def git_url = 'https://github.com/opencadc/core' diff --git a/cadc-util/src/main/java/org/opencadc/persist/Entity.java b/cadc-util/src/main/java/org/opencadc/persist/Entity.java index 379d5602..c7b6ea65 100644 --- a/cadc-util/src/main/java/org/opencadc/persist/Entity.java +++ b/cadc-util/src/main/java/org/opencadc/persist/Entity.java @@ -120,10 +120,12 @@ public abstract class Entity { private final String localPackage; public static boolean MCS_DEBUG = false; // way to much debug when true + public static byte[] ZERO_BYTE = new byte[] { (byte) 0 }; private final boolean digestFieldNames; private final boolean digestFieldNamesLowerCase; private final boolean truncateDateToSec; + private final boolean digestZeroByteAfterListItem; private UUID id; private Date lastModified; private URI metaChecksum; @@ -146,83 +148,42 @@ static final void assertNotNull(Class caller, String name, Object test) } /** - * Backwards compatible constructor: digestFieldNames==false. - * - * @param truncateDateToSec truncate Date values to seconds when converting to bytes for meta checksum calculation - * @deprecated hard code Entity(boolean, boolean, boolean) in model - */ - @Deprecated - protected Entity(boolean truncateDateToSec) { - this(truncateDateToSec, false, false); - } - - /** - * Backwards compatible constructor: digestFieldNames==false. - * - * @param id assign the specified Entity.id - * @param truncateDateToSec truncate Date values to seconds when converting to bytes for meta checksum calculation - * @deprecated hard code Entity(UUID, boolean, boolean, boolean) in model - */ - @Deprecated - protected Entity(UUID id, boolean truncateDateToSec) { - this(id, truncateDateToSec, false, false); - } - - /** - * Backwards compatible constructor: digestFieldNamesLowerCase==false. - * - * @param truncateDateToSec truncate Date values to seconds when converting to bytes for meta checksum calculation - * @param digestFieldNames when a field is not null (or collection is non-empty), include the field name in the - * metaChecksum calculation - */ - protected Entity(boolean truncateDateToSec, boolean digestFieldNames) { - this(truncateDateToSec, digestFieldNames, false); - } - - /** - * Backwards compatible constructor: digestFieldNamesLowerCase==false. - * - * @param id assign the specified Entity.id - * @param truncateDateToSec truncate Date values to seconds when converting to bytes for meta checksum calculation - * @param digestFieldNames when a field is not null (or collection is non-empty), include the field name in the - * metaChecksum calculation - */ - protected Entity(UUID id, boolean truncateDateToSec, boolean digestFieldNames) { - this(id, truncateDateToSec, digestFieldNames, false); - } - - /** - * Constructor.This creates a new entity with a random UUID. + * Constructor. This creates a new entity with a random UUID. * * @param truncateDateToSec truncate Date values to seconds when converting to bytes for meta checksum calculation * @param digestFieldNames when a field is not null (or collection is non-empty), include the field name in the * metaChecksum calculation * @param digestFieldNamesLowerCase convert field names to lower case before digesting + * @param digestZeroByteAfterListItem digest a single byte value 0 after item in a collection */ - protected Entity(boolean truncateDateToSec, boolean digestFieldNames, boolean digestFieldNamesLowerCase) { - this(UUID.randomUUID(), truncateDateToSec, digestFieldNames, digestFieldNamesLowerCase); + protected Entity(boolean truncateDateToSec, boolean digestFieldNames, boolean digestFieldNamesLowerCase, + boolean digestZeroByteAfterListItem) { + this(UUID.randomUUID(), truncateDateToSec, digestFieldNames, digestFieldNamesLowerCase, digestZeroByteAfterListItem); } /** * Constructor.This creates an entity with an existing UUID when reconstructing an instance. The - truncateDateToSec option should be used if instances of the model are to be serialized or stored - in a way that does not recover the exact timestamp to milliseconds. The digestFieldNames option - is needed for any model with "adjacent" fields that could contain the same value; this option - ensures that "moving" the value from one field to another will change the checksum by changing - the sequence of bytes that are digested. + * truncateDateToSec option should be used if instances of the model are to be serialized or stored + * in a way that does not recover the exact timestamp to milliseconds. The digestFieldNames option + * is needed for any model with "adjacent" fields that could contain the same value; this option + * ensures that "moving" the value from one field to another will change the checksum by changing + * the sequence of bytes that are digested. * * @param id unique ID value to assign/restore * @param truncateDateToSec truncate Date values to seconds when converting to bytes for meta checksum calculation * @param digestFieldNames when a field is not null (or collection is non-empty), include the field name in the * metaChecksum calculation * @param digestFieldNamesLowerCase convert field names to lower case before digesting + * @param digestZeroByteAfterListItem digest a single byte value 0 after item in a collection */ - protected Entity(UUID id, boolean truncateDateToSec, boolean digestFieldNames, boolean digestFieldNamesLowerCase) { + protected Entity(UUID id, boolean truncateDateToSec, boolean digestFieldNames, boolean digestFieldNamesLowerCase, + boolean digestZeroByteAfterListItem) { Entity.assertNotNull(Entity.class, "id", id); this.id = id; this.truncateDateToSec = truncateDateToSec; this.digestFieldNames = digestFieldNames; this.digestFieldNamesLowerCase = digestFieldNamesLowerCase; + this.digestZeroByteAfterListItem = digestZeroByteAfterListItem; this.localPackage = this.getClass().getPackage().getName(); } @@ -433,11 +394,16 @@ protected final void calcMetaChecksum(Class c, Object o, MessageDigestWrapper di String cf = f.getDeclaringClass().getSimpleName() + "." + f.getName(); f.setAccessible(true); Object fo = f.get(o); + if (fo != null) { Class ac = fo.getClass(); - if (ac.isEnum() || PrimitiveWrapper.class.isAssignableFrom(ac)) { + if (fo instanceof PrimitiveWrapper) { + PrimitiveWrapper pw = (PrimitiveWrapper) fo; + fo = pw.getValue(); + ac = fo.getClass(); + } + if (ac.isEnum()) { try { - log.warn("unwrap: " + ac.getSimpleName() + ".getValue()"); Method m = ac.getMethod("getValue"); Object val = m.invoke(fo); digest.update(primitiveValueToBytes(val, cf)); @@ -454,14 +420,32 @@ protected final void calcMetaChecksum(Class c, Object o, MessageDigestWrapper di if (digestFieldNames && num < digest.getNumBytes()) { digest.update(fieldNameToBytes(cf)); // field name } + } else if (ac.isArray()) { + Iterator iter = new ArrayIterator(fo); + if (iter.hasNext()) { + // arrays of primitive only + while (iter.hasNext()) { + Object co = iter.next(); + Class cc = co.getClass(); + digest.update(primitiveValueToBytes(co, cf)); + } + if (digestFieldNames) { + digest.update(fieldNameToBytes(cf)); // field name + } + } } else if (fo instanceof Collection) { Collection stuff = (Collection) fo; if (!stuff.isEmpty()) { - Iterator i = stuff.iterator(); - while (i.hasNext()) { - Object co = i.next(); + Iterator iter = stuff.iterator(); + while (iter.hasNext()) { + Object co = iter.next(); Class cc = co.getClass(); - if (cc.isEnum() || PrimitiveWrapper.class.isAssignableFrom(cc)) { + if (co instanceof PrimitiveWrapper) { + PrimitiveWrapper cpo = (PrimitiveWrapper) co; + co = cpo.getValue(); + cc = co.getClass(); + } + if (cc.isEnum()) { try { Method m = cc.getMethod("getValue"); Object val = m.invoke(co); @@ -472,9 +456,23 @@ protected final void calcMetaChecksum(Class c, Object o, MessageDigestWrapper di } else if (isDataModelClass(cc)) { // depth-first recursion calcMetaChecksum(cc, co, digest); + } else if (cc.isArray()) { + // use case: Interval -> Object[] + Iterator ai = new ArrayIterator(co); + if (ai.hasNext()) { + // arrays of primitive only + while (ai.hasNext()) { + Object ico = ai.next(); + Class icc = co.getClass(); + digest.update(primitiveValueToBytes(ico, cf)); + } + } } else { digest.update(primitiveValueToBytes(co, cf)); } + if (digestZeroByteAfterListItem) { + digest.update(ZERO_BYTE); + } } if (digestFieldNames) { digest.update(fieldNameToBytes(cf)); // field name @@ -496,6 +494,28 @@ protected final void calcMetaChecksum(Class c, Object o, MessageDigestWrapper di } } + private static class ArrayIterator implements Iterator { + private Object arr; + private int len; + private int cur; + + public ArrayIterator(Object arr) { + this.arr = arr; + this.len = java.lang.reflect.Array.getLength(arr); + this.cur = 0; + } + + @Override + public boolean hasNext() { + return cur < len; + } + + @Override + public Object next() { + return java.lang.reflect.Array.get(arr, cur++); + } + } + public static class MessageDigestWrapper { private MessageDigest digest; private int numBytes = 0; @@ -677,7 +697,7 @@ protected byte[] primitiveValueToBytes(Object o, String name) { return ret; } - throw new UnsupportedOperationException("unexpected primitive/value type: " + o.getClass().getName()); + throw new UnsupportedOperationException("unexpected primitive/value type: " + o.getClass().getName() + " field: " + name); } protected byte[] fieldNameToBytes(String name) { diff --git a/cadc-util/src/test/java/ca/nrc/cadc/date/DateUtilTest.java b/cadc-util/src/test/java/ca/nrc/cadc/date/DateUtilTest.java index 1746612e..463c3b69 100644 --- a/cadc-util/src/test/java/ca/nrc/cadc/date/DateUtilTest.java +++ b/cadc-util/src/test/java/ca/nrc/cadc/date/DateUtilTest.java @@ -289,6 +289,16 @@ public void testFromModifiedJulianDateToISO8601Date() Assert.fail("unexpected exception: " + unexpected); throw unexpected; } + + DateFormat df = DateUtil.getDateFormat(DateUtil.IVOA_DATE_FORMAT, DateUtil.UTC); + String str = "2026-03-08T02:02:41.056"; + Date orig = df.parse(str); + double mjd = DateUtil.toModifiedJulianDate(orig); + Date actual = DateUtil.fromModifiedJulianDate(mjd); + log.info(str + " -> " + df.format(orig) + " -> " + mjd + " -> " + df.format(actual)); + Assert.assertEquals(orig, actual); + + } @Test diff --git a/cadc-util/src/test/java/org/opencadc/persist/EntityTest.java b/cadc-util/src/test/java/org/opencadc/persist/EntityTest.java index 18d9dc66..056640c5 100644 --- a/cadc-util/src/test/java/org/opencadc/persist/EntityTest.java +++ b/cadc-util/src/test/java/org/opencadc/persist/EntityTest.java @@ -85,7 +85,7 @@ public class EntityTest { static { Log4jInit.setLevel("org.opencadc.persist", Level.DEBUG); - // this actually controls the large amoutn of debug output from checksum + // this actually controls the large amount of debug output from checksum // algorithm, but it effects the whole jvm so only enable when running // these tests specificially and looking at output //Entity.MCS_DEBUG = true; @@ -109,41 +109,41 @@ public void testTemplate() { @Test public void testEntity() { // base: the cadc-inventory-0.x configuration - doEntityTest(false, false, false); - doNewVersionTest(false, false, false); + doEntityTest(false, false, false, false); + doNewVersionTest(false, false, false, false); } @Test public void testEntityTruncateDates() { // the caom2-2.4 configuration - doEntityTest(true, false, false); - doNewVersionTest(true, false, false); + doEntityTest(true, false, false, false); + doNewVersionTest(true, false, false, false); } @Test public void testEntityDigestFieldNames() { // the cadc-vos-2.x configuration - doEntityTest(false, true, false); - doNewVersionTest(false, true, false); + doEntityTest(false, true, false, false); + doNewVersionTest(false, true, false, false); } @Test public void testEntityDigestFieldNamesLower() { - // the cadc-vos-2.x configuration - doEntityTest(false, true, true); - doNewVersionTest(false, true, true); + // the caom-2.5 configuration + doEntityTest(false, true, true, false); + doNewVersionTest(false, true, true, false); } @Test public void testEntitySafeMode() { // no known use, but truncateDates and digestFieldNames is the safest mode - doEntityTest(true, true, true); - doNewVersionTest(true, true, true); + doEntityTest(true, true, true, true); + doNewVersionTest(true, true, true, false); } - private void doEntityTest(boolean trunc, boolean dig, boolean digL) { + private void doEntityTest(boolean trunc, boolean dig, boolean digL, boolean digZB) { try { - SampleEntity sample = new SampleEntity("name-of-this-entity", trunc, dig, digL); + SampleEntity sample = new SampleEntity("name-of-this-entity", trunc, dig, digL, digZB); log.info("created: " + sample); URI mcs1 = sample.computeMetaChecksum(MessageDigest.getInstance("MD5")); @@ -173,12 +173,23 @@ private void doEntityTest(boolean trunc, boolean dig, boolean digL) { Assert.assertNotEquals(mcs6, mcs7); // set of string - sample.strList.add("foo"); + sample.strList.add("abc"); URI mcs8 = sample.computeMetaChecksum(MessageDigest.getInstance("MD5")); Assert.assertNotEquals(mcs7, mcs8); - sample.strList.add("bar"); + sample.strList.add("def"); URI mcs9 = sample.computeMetaChecksum(MessageDigest.getInstance("MD5")); Assert.assertNotEquals(mcs8, mcs9); + // list-item subtleness + sample.strList.clear(); + sample.strList.add("abcd"); + sample.strList.add("ef"); + log.warn("checking digZB..."); + URI mcs9b = sample.computeMetaChecksum(MessageDigest.getInstance("MD5")); + if (digZB) { + Assert.assertNotEquals(mcs9, mcs9b); + } else { + Assert.assertEquals(mcs9, mcs9b); + } // revert to 7 sample.strList.clear(); URI mcs10 = sample.computeMetaChecksum(MessageDigest.getInstance("MD5")); @@ -193,11 +204,11 @@ private void doEntityTest(boolean trunc, boolean dig, boolean digL) { Assert.assertNotEquals(mcs7, mcs12); // entities do not get included in metaChecksum - sample.children.add(new SampleEntity("flibble", trunc, dig, digL)); + sample.children.add(new SampleEntity("flibble", trunc, dig, digL, digZB)); URI tcs1 = sample.computeMetaChecksum(MessageDigest.getInstance("MD5")); Assert.assertEquals(mcs12, tcs1); - sample.relation = new SampleEntity("flibble", trunc, dig, digL); + sample.relation = new SampleEntity("flibble", trunc, dig, digL, digZB); mcs11 = sample.computeMetaChecksum(MessageDigest.getInstance("MD5")); Assert.assertEquals(mcs12, mcs11); @@ -219,13 +230,13 @@ private void doEntityTest(boolean trunc, boolean dig, boolean digL) { } // also doubles as a sub-class/extension test - private void doNewVersionTest(boolean trunc, boolean dig, boolean digL) { + private void doNewVersionTest(boolean trunc, boolean dig, boolean digL, boolean digZB) { try { - SampleEntity v1 = new SampleEntity("name-of-this-entity", trunc, dig, digL); + SampleEntity v1 = new SampleEntity("name-of-this-entity", trunc, dig, digL, digZB); log.info("created: " + v1); URI mcs1 = v1.computeMetaChecksum(MessageDigest.getInstance("MD5")); - SampleEntityV2 v2 = new SampleEntityV2(v1.getID(), v1.getName(), trunc, dig, digL); + SampleEntityV2 v2 = new SampleEntityV2(v1.getID(), v1.getName(), trunc, dig, digL, digZB); log.info("created: " + v1); URI mcs2 = v2.computeMetaChecksum(MessageDigest.getInstance("MD5")); @@ -239,7 +250,7 @@ private void doNewVersionTest(boolean trunc, boolean dig, boolean digL) { @Test public void testNonState() { try { - SampleEntity sample = new SampleEntity("name-of-this-entity", false, false, false); + SampleEntity sample = new SampleEntity("name-of-this-entity", false, false, false, false); log.info("created: " + sample); URI mcs1 = sample.computeMetaChecksum(MessageDigest.getInstance("MD5")); diff --git a/cadc-util/src/test/java/org/opencadc/persist/EntityVisitorTest.java b/cadc-util/src/test/java/org/opencadc/persist/EntityVisitorTest.java index c269d24d..60cd5e60 100644 --- a/cadc-util/src/test/java/org/opencadc/persist/EntityVisitorTest.java +++ b/cadc-util/src/test/java/org/opencadc/persist/EntityVisitorTest.java @@ -91,7 +91,7 @@ public EntityVisitorTest() { @Test public void testLogVisitor() { - SampleEntity e = new SampleEntity("foo", false, true, true); + SampleEntity e = new SampleEntity("foo", false, true, true, true); e.dateVal = null; e.doubleVal = 2.0; e.longVal = 123L; diff --git a/cadc-util/src/test/java/org/opencadc/persist/SampleEntity.java b/cadc-util/src/test/java/org/opencadc/persist/SampleEntity.java index 742ae140..dbb62067 100644 --- a/cadc-util/src/test/java/org/opencadc/persist/SampleEntity.java +++ b/cadc-util/src/test/java/org/opencadc/persist/SampleEntity.java @@ -102,13 +102,15 @@ public class SampleEntity extends Entity implements Comparable { public transient String transientVal; - public SampleEntity(String name, boolean truncateDateToSec, boolean digestFieldNames, boolean digestFieldNamesLowerCase) { - super(truncateDateToSec, digestFieldNames, digestFieldNamesLowerCase); + public SampleEntity(String name, boolean truncateDateToSec, boolean digestFieldNames, boolean digestFieldNamesLowerCase, + boolean digestZeroByteAfterListItem) { + super(truncateDateToSec, digestFieldNames, digestFieldNamesLowerCase, digestZeroByteAfterListItem); this.name = name; } - public SampleEntity(UUID id, String name, boolean truncateDateToSec, boolean digestFieldNames, boolean digestFieldNamesLowerCase) { - super(id, truncateDateToSec, digestFieldNames, digestFieldNamesLowerCase); + public SampleEntity(UUID id, String name, boolean truncateDateToSec, boolean digestFieldNames, boolean digestFieldNamesLowerCase, + boolean digestZeroByteAfterListItem) { + super(id, truncateDateToSec, digestFieldNames, digestFieldNamesLowerCase, digestZeroByteAfterListItem); this.name = name; } diff --git a/cadc-util/src/test/java/org/opencadc/persist/SampleEntityV2.java b/cadc-util/src/test/java/org/opencadc/persist/SampleEntityV2.java index bd6ad14c..5e34f861 100644 --- a/cadc-util/src/test/java/org/opencadc/persist/SampleEntityV2.java +++ b/cadc-util/src/test/java/org/opencadc/persist/SampleEntityV2.java @@ -80,8 +80,8 @@ public class SampleEntityV2 extends SampleEntity { public Integer optionalInt; public String optionalString; - public SampleEntityV2(UUID id, String name, boolean trunc, boolean dig, boolean digL) { - super(id, name, trunc, dig, digL); + public SampleEntityV2(UUID id, String name, boolean trunc, boolean dig, boolean digL, boolean digZB) { + super(id, name, trunc, dig, digL, digZB); } public String toString() { From 183c4e1e24a4e1e57e25acad1273d18a9527d583 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Thu, 16 Apr 2026 12:52:35 -0700 Subject: [PATCH 2/6] rename PrimitiveWrapper.getValue() to getWrappedValue() former method name often collides with existing use --- cadc-util/src/main/java/org/opencadc/persist/Entity.java | 4 ++-- .../src/main/java/org/opencadc/persist/PrimitiveWrapper.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cadc-util/src/main/java/org/opencadc/persist/Entity.java b/cadc-util/src/main/java/org/opencadc/persist/Entity.java index c7b6ea65..53b7bfc4 100644 --- a/cadc-util/src/main/java/org/opencadc/persist/Entity.java +++ b/cadc-util/src/main/java/org/opencadc/persist/Entity.java @@ -399,7 +399,7 @@ protected final void calcMetaChecksum(Class c, Object o, MessageDigestWrapper di Class ac = fo.getClass(); if (fo instanceof PrimitiveWrapper) { PrimitiveWrapper pw = (PrimitiveWrapper) fo; - fo = pw.getValue(); + fo = pw.getWrappedValue(); ac = fo.getClass(); } if (ac.isEnum()) { @@ -442,7 +442,7 @@ protected final void calcMetaChecksum(Class c, Object o, MessageDigestWrapper di Class cc = co.getClass(); if (co instanceof PrimitiveWrapper) { PrimitiveWrapper cpo = (PrimitiveWrapper) co; - co = cpo.getValue(); + co = cpo.getWrappedValue(); cc = co.getClass(); } if (cc.isEnum()) { diff --git a/cadc-util/src/main/java/org/opencadc/persist/PrimitiveWrapper.java b/cadc-util/src/main/java/org/opencadc/persist/PrimitiveWrapper.java index 93acdd68..f06207a2 100644 --- a/cadc-util/src/main/java/org/opencadc/persist/PrimitiveWrapper.java +++ b/cadc-util/src/main/java/org/opencadc/persist/PrimitiveWrapper.java @@ -75,5 +75,5 @@ * @author pdowler */ public interface PrimitiveWrapper { - public Object getValue(); + public Object getWrappedValue(); } From d65d743fa9bca445782cc6cd58905543eec77d04 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Thu, 16 Apr 2026 13:28:12 -0700 Subject: [PATCH 3/6] document PrimitiveWrapper to return a specific set of types for arrays, only support primtivve arrays (double[] and long[], no Object[]) check and unwrap in visit() --- .../java/org/opencadc/persist/Entity.java | 49 +++++++++---------- .../opencadc/persist/PrimitiveWrapper.java | 7 +++ 2 files changed, 29 insertions(+), 27 deletions(-) diff --git a/cadc-util/src/main/java/org/opencadc/persist/Entity.java b/cadc-util/src/main/java/org/opencadc/persist/Entity.java index 53b7bfc4..ffd59726 100644 --- a/cadc-util/src/main/java/org/opencadc/persist/Entity.java +++ b/cadc-util/src/main/java/org/opencadc/persist/Entity.java @@ -146,7 +146,7 @@ static final void assertNotNull(Class caller, String name, Object test) throw new IllegalArgumentException("invalid " + caller.getSimpleName() + "." + name + ": null"); } } - + /** * Constructor. This creates a new entity with a random UUID. * @@ -297,7 +297,12 @@ private void visitImpl(Class c, Object o, EntityVisitor ev) { ev.visitNull(cf); } else { Class ac = fo.getClass(); - if (ac.isEnum() || PrimitiveWrapper.class.isAssignableFrom(ac)) { + if (fo instanceof PrimitiveWrapper) { + PrimitiveWrapper pw = (PrimitiveWrapper) fo; + fo = pw.getWrappedValue(); + ac = fo.getClass(); + } + if (ac.isEnum()) { try { log.warn("unwrap: " + ac.getSimpleName() + ".getValue()"); Method m = ac.getMethod("getValue"); @@ -318,7 +323,12 @@ private void visitImpl(Class c, Object o, EntityVisitor ev) { while (i.hasNext()) { Object co = i.next(); Class cc = co.getClass(); - if (cc.isEnum() || PrimitiveWrapper.class.isAssignableFrom(cc)) { + if (co instanceof PrimitiveWrapper) { + PrimitiveWrapper cpo = (PrimitiveWrapper) co; + co = cpo.getWrappedValue(); + cc = co.getClass(); + } + if (cc.isEnum()) { try { Method m = cc.getMethod("getValue"); Object val = m.invoke(co); @@ -398,6 +408,7 @@ protected final void calcMetaChecksum(Class c, Object o, MessageDigestWrapper di if (fo != null) { Class ac = fo.getClass(); if (fo instanceof PrimitiveWrapper) { + // unwrap PrimitiveWrapper pw = (PrimitiveWrapper) fo; fo = pw.getWrappedValue(); ac = fo.getClass(); @@ -420,19 +431,6 @@ protected final void calcMetaChecksum(Class c, Object o, MessageDigestWrapper di if (digestFieldNames && num < digest.getNumBytes()) { digest.update(fieldNameToBytes(cf)); // field name } - } else if (ac.isArray()) { - Iterator iter = new ArrayIterator(fo); - if (iter.hasNext()) { - // arrays of primitive only - while (iter.hasNext()) { - Object co = iter.next(); - Class cc = co.getClass(); - digest.update(primitiveValueToBytes(co, cf)); - } - if (digestFieldNames) { - digest.update(fieldNameToBytes(cf)); // field name - } - } } else if (fo instanceof Collection) { Collection stuff = (Collection) fo; if (!stuff.isEmpty()) { @@ -441,6 +439,7 @@ protected final void calcMetaChecksum(Class c, Object o, MessageDigestWrapper di Object co = iter.next(); Class cc = co.getClass(); if (co instanceof PrimitiveWrapper) { + // unwrap PrimitiveWrapper cpo = (PrimitiveWrapper) co; co = cpo.getWrappedValue(); cc = co.getClass(); @@ -456,17 +455,6 @@ protected final void calcMetaChecksum(Class c, Object o, MessageDigestWrapper di } else if (isDataModelClass(cc)) { // depth-first recursion calcMetaChecksum(cc, co, digest); - } else if (cc.isArray()) { - // use case: Interval -> Object[] - Iterator ai = new ArrayIterator(co); - if (ai.hasNext()) { - // arrays of primitive only - while (ai.hasNext()) { - Object ico = ai.next(); - Class icc = co.getClass(); - digest.update(primitiveValueToBytes(ico, cf)); - } - } } else { digest.update(primitiveValueToBytes(co, cf)); } @@ -684,6 +672,13 @@ protected byte[] primitiveValueToBytes(Object o, String name) { byte[] b = HexUtil.toBytes(Double.doubleToLongBits(da[i])); // IEEE754 double System.arraycopy(b, 0, ret, i * 8, 8); } + } else if (o instanceof long[]) { + long[] da = (long[]) o; + ret = new byte[8 * da.length]; + for (int i = 0; i < da.length; i++) { + byte[] b = HexUtil.toBytes(da[i]); + System.arraycopy(b, 0, ret, i * 8, 8); + } } if (ret != null) { diff --git a/cadc-util/src/main/java/org/opencadc/persist/PrimitiveWrapper.java b/cadc-util/src/main/java/org/opencadc/persist/PrimitiveWrapper.java index f06207a2..2de403e8 100644 --- a/cadc-util/src/main/java/org/opencadc/persist/PrimitiveWrapper.java +++ b/cadc-util/src/main/java/org/opencadc/persist/PrimitiveWrapper.java @@ -75,5 +75,12 @@ * @author pdowler */ public interface PrimitiveWrapper { + /** + * Unwrap the inner state and return it. The return value can be a primitive, + * immutable class (String, URI, Double, Long, etc), a primitive array (e.g. double[]), + * or a java.util.Collection (possibly empty). + * + * @return the wrapped value + */ public Object getWrappedValue(); } From 98c1ccaff3f71abf683f702ecf7dffaf12ad122c Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 22 Apr 2026 11:05:55 -0700 Subject: [PATCH 4/6] augment EntityVisitor interface to support extra validation add visitChild* methods and update visitImpl to call them for child entities in model --- .../java/org/opencadc/persist/Entity.java | 35 ++++++++++++++----- .../org/opencadc/persist/EntityVisitor.java | 26 +++++++++++++- .../java/org/opencadc/persist/EntityTest.java | 2 +- .../opencadc/persist/EntityVisitorTest.java | 30 ++++++++++++++-- .../org/opencadc/persist/SampleEntity.java | 4 ++- 5 files changed, 83 insertions(+), 14 deletions(-) diff --git a/cadc-util/src/main/java/org/opencadc/persist/Entity.java b/cadc-util/src/main/java/org/opencadc/persist/Entity.java index ffd59726..f3986af0 100644 --- a/cadc-util/src/main/java/org/opencadc/persist/Entity.java +++ b/cadc-util/src/main/java/org/opencadc/persist/Entity.java @@ -288,13 +288,13 @@ private void visitImpl(Class c, Object o, EntityVisitor ev) { SortedSet fields = getStateFields(c); for (Field f : fields) { - String cf = f.getDeclaringClass().getSimpleName() + "." + f.getName(); + String vodmlID = f.getDeclaringClass().getSimpleName() + "." + f.getName(); f.setAccessible(true); Object fo = f.get(o); if (fo == null) { - ev.visitNull(cf); + ev.visitNull(vodmlID); } else { Class ac = fo.getClass(); if (fo instanceof PrimitiveWrapper) { @@ -307,7 +307,7 @@ private void visitImpl(Class c, Object o, EntityVisitor ev) { log.warn("unwrap: " + ac.getSimpleName() + ".getValue()"); Method m = ac.getMethod("getValue"); Object val = m.invoke(fo); - ev.visitLeaf(cf, val); + ev.visitLeaf(vodmlID, val); } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException ex) { throw new RuntimeException("BUG - enum " + ac.getName() + " does not have getValue()", ex); } @@ -315,7 +315,7 @@ private void visitImpl(Class c, Object o, EntityVisitor ev) { // depth-first recursion visitImpl(ac, fo, ev); // visit intermediate DM class - ev.visitNode(cf, fo); + ev.visitNode(vodmlID, fo); } else if (fo instanceof Collection) { Collection stuff = (Collection) fo; if (!stuff.isEmpty()) { @@ -332,7 +332,7 @@ private void visitImpl(Class c, Object o, EntityVisitor ev) { try { Method m = cc.getMethod("getValue"); Object val = m.invoke(co); - ev.visitLeaf(cf, val); + ev.visitLeaf(vodmlID, val); } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException ex) { throw new RuntimeException("BUG", ex); } @@ -340,19 +340,36 @@ private void visitImpl(Class c, Object o, EntityVisitor ev) { // depth-first recursion visitImpl(cc, co, ev); // visit intermediate DM class - ev.visitNode(cf, co); + ev.visitNode(vodmlID, co); } else { - ev.visitLeaf(cf, co); + ev.visitLeaf(vodmlID, co); } } } - ev.visitCollection(cf, stuff); + ev.visitCollection(vodmlID, stuff); } else { - ev.visitLeaf(cf, fo); + ev.visitLeaf(vodmlID, fo); } } } + SortedSet cfields = getChildFields(c); + for (Field cf : cfields) { + String vodmlID = cf.getDeclaringClass().getSimpleName() + "." + cf.getName(); + cf.setAccessible(true); + Object cfo = cf.get(o); + if (cfo == null) { + ev.visitChildNull(vodmlID); + } else if (cfo instanceof Collection) { + // child fields always collection, never null + Collection stuff = (Collection) cfo; + ev.visitChildCollection(vodmlID, stuff); + } else { + Entity e = (Entity) cfo; + ev.visitChildEntity(vodmlID, e); + } + } + } catch (IllegalAccessException bug) { throw new RuntimeException("Unable to calculate metaChecksum for class " + c.getName(), bug); } diff --git a/cadc-util/src/main/java/org/opencadc/persist/EntityVisitor.java b/cadc-util/src/main/java/org/opencadc/persist/EntityVisitor.java index 8707bd58..a145beb6 100644 --- a/cadc-util/src/main/java/org/opencadc/persist/EntityVisitor.java +++ b/cadc-util/src/main/java/org/opencadc/persist/EntityVisitor.java @@ -99,8 +99,32 @@ public interface EntityVisitor { public void visitNode(String vodmlID, Object val); /** - * Visit a null value. This could be a node or a primitive. + * Visit a null value. This could be a node or a leaf. * @param vodmlID the {declaringClass}.{fieldName} aka vodml-id */ public void visitNull(String vodmlID); + + /** + * Visit a collection of child entity(s). + * + * @param vodmlID the {declaringClass}.{fieldName} aka vodml-id + * @param val the collection + */ + public void visitChildCollection(String vodmlID, Collection val); + + /** + * Visit a null child entity value. + * + * @param vodmlID the {declaringClass}.{fieldName} aka vodml-id + */ + public void visitChildNull(String vodmlID); + + /** + * Visit a direct child entity. There is no recursion into the structure of + * the child entity. + * + * @param vodmlID the {declaringClass}.{fieldName} aka vodml-id + * @param val the entity + */ + public void visitChildEntity(String vodmlID, Entity val); } diff --git a/cadc-util/src/test/java/org/opencadc/persist/EntityTest.java b/cadc-util/src/test/java/org/opencadc/persist/EntityTest.java index 056640c5..88cb9680 100644 --- a/cadc-util/src/test/java/org/opencadc/persist/EntityTest.java +++ b/cadc-util/src/test/java/org/opencadc/persist/EntityTest.java @@ -208,7 +208,7 @@ private void doEntityTest(boolean trunc, boolean dig, boolean digL, boolean digZ URI tcs1 = sample.computeMetaChecksum(MessageDigest.getInstance("MD5")); Assert.assertEquals(mcs12, tcs1); - sample.relation = new SampleEntity("flibble", trunc, dig, digL, digZB); + sample.child1 = new SampleEntity("flibble", trunc, dig, digL, digZB); mcs11 = sample.computeMetaChecksum(MessageDigest.getInstance("MD5")); Assert.assertEquals(mcs12, mcs11); diff --git a/cadc-util/src/test/java/org/opencadc/persist/EntityVisitorTest.java b/cadc-util/src/test/java/org/opencadc/persist/EntityVisitorTest.java index 60cd5e60..0b622dc8 100644 --- a/cadc-util/src/test/java/org/opencadc/persist/EntityVisitorTest.java +++ b/cadc-util/src/test/java/org/opencadc/persist/EntityVisitorTest.java @@ -99,6 +99,7 @@ public void testLogVisitor() { e.nestedSet.add(new SampleEntity.Nested("abc")); e.nestedSet.add(new SampleEntity.Nested("def")); e.strList.add("foo"); + e.child1 = new SampleEntity("foo.child1", false, true, true, true); log.info("testLogVisitor: START"); LogVisitor v = new LogVisitor(); @@ -108,16 +109,25 @@ public void testLogVisitor() { Assert.assertEquals("numLeaf", 9, v.numLeaf); Assert.assertEquals("numNode", 3, v.numNode); // nested, nestedSet, emptySet Assert.assertEquals("numNull", 7, v.numNull); + Assert.assertEquals("numCol", 3, v.numCol); + Assert.assertEquals("numChildCol", 1, v.numChildCol); + Assert.assertEquals("numChildEntity", 1, v.numChildEntity); + Assert.assertEquals("numChildNull", 2, v.numChildNull); } private class LogVisitor implements EntityVisitor { int numLeaf = 0; int numNode = 0; int numNull = 0; + int numCol = 0; + int numChildCol = 0; + int numChildNull = 0; + int numChildEntity = 0; @Override public void visitCollection(String vodmlID, Collection val) { System.out.println("visit collection: " + vodmlID + " " + val.size()); + numCol++; } @Override @@ -137,7 +147,23 @@ public void visitNull(String vodmlID) { System.out.println("visit null: " + vodmlID); numNull++; } - - + + @Override + public void visitChildCollection(String vodmlID, Collection val) { + System.out.println("visit collection: " + vodmlID); + numChildCol++; + } + + @Override + public void visitChildNull(String vodmlID) { + System.out.println("visit child null: " + vodmlID); + numChildNull++; + } + + @Override + public void visitChildEntity(String vodmlID, Entity val) { + System.out.println("visit child entity: " + vodmlID); + numChildEntity++; + } } } diff --git a/cadc-util/src/test/java/org/opencadc/persist/SampleEntity.java b/cadc-util/src/test/java/org/opencadc/persist/SampleEntity.java index dbb62067..b50b94ef 100644 --- a/cadc-util/src/test/java/org/opencadc/persist/SampleEntity.java +++ b/cadc-util/src/test/java/org/opencadc/persist/SampleEntity.java @@ -97,7 +97,9 @@ public class SampleEntity extends Entity implements Comparable { // not included public Set children = new TreeSet<>(); - public SampleEntity relation; + public SampleEntity child1; + public SampleEntity child2; + public SampleEntity child3; public static String staticVal; public transient String transientVal; From 87448ea8e1943c65b5e13da542eb7f57e265b4c7 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 22 Apr 2026 11:11:47 -0700 Subject: [PATCH 5/6] remove unused ArrayIterator --- .../java/org/opencadc/persist/Entity.java | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/cadc-util/src/main/java/org/opencadc/persist/Entity.java b/cadc-util/src/main/java/org/opencadc/persist/Entity.java index f3986af0..1db14f52 100644 --- a/cadc-util/src/main/java/org/opencadc/persist/Entity.java +++ b/cadc-util/src/main/java/org/opencadc/persist/Entity.java @@ -499,28 +499,6 @@ protected final void calcMetaChecksum(Class c, Object o, MessageDigestWrapper di } } - private static class ArrayIterator implements Iterator { - private Object arr; - private int len; - private int cur; - - public ArrayIterator(Object arr) { - this.arr = arr; - this.len = java.lang.reflect.Array.getLength(arr); - this.cur = 0; - } - - @Override - public boolean hasNext() { - return cur < len; - } - - @Override - public Object next() { - return java.lang.reflect.Array.get(arr, cur++); - } - } - public static class MessageDigestWrapper { private MessageDigest digest; private int numBytes = 0; From d12015abebcb4124f35490121cdd09311ec20843 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 1 May 2026 10:52:21 -0700 Subject: [PATCH 6/6] include and deprecate backwards compat Entity ctors for specific OpenCADC libraries --- .../java/org/opencadc/persist/Entity.java | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/cadc-util/src/main/java/org/opencadc/persist/Entity.java b/cadc-util/src/main/java/org/opencadc/persist/Entity.java index 1db14f52..d2c1084f 100644 --- a/cadc-util/src/main/java/org/opencadc/persist/Entity.java +++ b/cadc-util/src/main/java/org/opencadc/persist/Entity.java @@ -146,6 +146,30 @@ static final void assertNotNull(Class caller, String name, Object test) throw new IllegalArgumentException("invalid " + caller.getSimpleName() + "." + name + ": null"); } } + + // backwards compat ctors for cadc-vos + @Deprecated + protected Entity(boolean truncateDateToSec, boolean digestFieldNames) { + this(truncateDateToSec, digestFieldNames, false, false); + } + + @Deprecated + protected Entity(UUID id, boolean truncateDateToSec, boolean digestFieldNames) { + this(id, truncateDateToSec, digestFieldNames, false, false); + } + // end: cadc-vos + + // backwards compat ctor for cadc-inventory + @Deprecated + protected Entity(boolean truncateDateToSec, boolean digestFieldNames, boolean digestFieldNamesLowerCase) { + this(truncateDateToSec, digestFieldNames, digestFieldNamesLowerCase, false); + } + + @Deprecated + protected Entity(UUID id, boolean truncateDateToSec, boolean digestFieldNames, boolean digestFieldNamesLowerCase) { + this(id, truncateDateToSec, digestFieldNames, digestFieldNamesLowerCase, false); + } + // end: cadc-inventory /** * Constructor. This creates a new entity with a random UUID.