From dab0eab4498b6095440ac3d1a3b19af93f79c0a1 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 7 Apr 2026 11:43:40 -0700 Subject: [PATCH 1/7] cadc-dali: add TableReader interface to enable plugin readers --- cadc-dali-parquet/build.gradle | 4 +- .../dali/tables/parquet/ParquetReader.java | 8 +- cadc-dali/build.gradle | 2 +- .../ca/nrc/cadc/dali/tables/TableReader.java | 90 +++++++++++++++++++ .../dali/tables/votable/VOTableReader.java | 4 +- 5 files changed, 98 insertions(+), 10 deletions(-) create mode 100644 cadc-dali/src/main/java/ca/nrc/cadc/dali/tables/TableReader.java diff --git a/cadc-dali-parquet/build.gradle b/cadc-dali-parquet/build.gradle index 8436fecc..ed3c6f2a 100644 --- a/cadc-dali-parquet/build.gradle +++ b/cadc-dali-parquet/build.gradle @@ -15,14 +15,14 @@ sourceCompatibility = 11 group = 'org.opencadc' -version = '0.5.5' +version = '0.5.6' description = 'OpenCADC VO-parquet library' def git_url = 'https://github.com/opencadc/dal' dependencies { implementation 'org.opencadc:cadc-util:[1.12.10,)' - implementation 'org.opencadc:cadc-dali:[1.2.29,)' + implementation 'org.opencadc:cadc-dali:[1.2.30,)' implementation 'org.apache.hadoop:hadoop-common:[3.3.6,)' implementation 'org.apache.parquet:parquet-hadoop:[1.13.1,)' diff --git a/cadc-dali-parquet/src/main/java/ca/nrc/cadc/dali/tables/parquet/ParquetReader.java b/cadc-dali-parquet/src/main/java/ca/nrc/cadc/dali/tables/parquet/ParquetReader.java index 5cc1c6c6..da88d9dd 100644 --- a/cadc-dali-parquet/src/main/java/ca/nrc/cadc/dali/tables/parquet/ParquetReader.java +++ b/cadc-dali-parquet/src/main/java/ca/nrc/cadc/dali/tables/parquet/ParquetReader.java @@ -69,8 +69,8 @@ package ca.nrc.cadc.dali.tables.parquet; +import ca.nrc.cadc.dali.tables.TableReader; import static ca.nrc.cadc.dali.tables.parquet.ParquetWriter.IVOA_VOTABLE_PARQUET_CONTENT_KEY; - import ca.nrc.cadc.dali.tables.parquet.io.RandomSeekableInputFile; import ca.nrc.cadc.dali.tables.parquet.readerhelper.ParquetTableData; import ca.nrc.cadc.dali.tables.votable.VOTableDocument; @@ -78,7 +78,6 @@ import ca.nrc.cadc.dali.tables.votable.VOTableReader; import ca.nrc.cadc.dali.tables.votable.VOTableResource; import ca.nrc.cadc.dali.tables.votable.VOTableTable; - import ca.nrc.cadc.dali.util.ByteArrayFormat; import ca.nrc.cadc.dali.util.ByteFormat; import ca.nrc.cadc.dali.util.DoubleArrayFormat; @@ -97,17 +96,14 @@ import ca.nrc.cadc.io.MultiBufferIO; import ca.nrc.cadc.io.RandomAccessFile; import ca.nrc.cadc.io.RandomAccessSource; - import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; - import java.util.ArrayList; import java.util.List; import java.util.UUID; - import org.apache.log4j.Logger; import org.apache.parquet.hadoop.ParquetFileReader; import org.apache.parquet.hadoop.metadata.ParquetMetadata; @@ -121,7 +117,7 @@ * Parquet Reader - Reads parquet content and produces a VOTableDocument representation. * */ -public class ParquetReader { +public class ParquetReader implements TableReader { private static final Logger log = Logger.getLogger(ParquetReader.class); diff --git a/cadc-dali/build.gradle b/cadc-dali/build.gradle index 675f02d7..e2b3b693 100644 --- a/cadc-dali/build.gradle +++ b/cadc-dali/build.gradle @@ -15,7 +15,7 @@ sourceCompatibility = 11 group = 'org.opencadc' -version = '1.2.29' +version = '1.2.30' description = 'OpenCADC DALI library' def git_url = 'https://github.com/opencadc/dal' diff --git a/cadc-dali/src/main/java/ca/nrc/cadc/dali/tables/TableReader.java b/cadc-dali/src/main/java/ca/nrc/cadc/dali/tables/TableReader.java new file mode 100644 index 00000000..b84d3c2c --- /dev/null +++ b/cadc-dali/src/main/java/ca/nrc/cadc/dali/tables/TableReader.java @@ -0,0 +1,90 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2025. (c) 2025. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package ca.nrc.cadc.dali.tables; + +import ca.nrc.cadc.dali.tables.votable.VOTableDocument; +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; +import org.apache.log4j.Logger; + +/** + * + * @author pdowler + */ +public interface TableReader { + + /** + * Read a table from an InputStream and build a VOTable object. + * + * @param istream InputStream to read from. + * @return a VOTable object. + * @throws IOException is problem reading the InputStream. + */ + VOTableDocument read(InputStream istream) throws IOException; +} diff --git a/cadc-dali/src/main/java/ca/nrc/cadc/dali/tables/votable/VOTableReader.java b/cadc-dali/src/main/java/ca/nrc/cadc/dali/tables/votable/VOTableReader.java index 8ea8a495..a3173cdc 100644 --- a/cadc-dali/src/main/java/ca/nrc/cadc/dali/tables/votable/VOTableReader.java +++ b/cadc-dali/src/main/java/ca/nrc/cadc/dali/tables/votable/VOTableReader.java @@ -69,6 +69,7 @@ package ca.nrc.cadc.dali.tables.votable; +import ca.nrc.cadc.dali.tables.TableReader; import static ca.nrc.cadc.dali.tables.votable.VOTableWriter.SerializationType.BINARY; import static ca.nrc.cadc.dali.tables.votable.VOTableWriter.SerializationType.BINARY2; @@ -106,7 +107,7 @@ * * @author pdowler */ -public class VOTableReader { +public class VOTableReader implements TableReader { private static final Logger log = Logger.getLogger(VOTableReader.class); @@ -206,6 +207,7 @@ public VOTableDocument read(String xml) * @return a VOTable object. * @throws IOException is problem reading the InputStream. */ + @Override public VOTableDocument read(InputStream istream) throws IOException { Reader reader = new BufferedReader(new InputStreamReader(istream, "UTF-8")); From b16201a71a036232949b53c098587a4fd6aba2c9 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 7 Apr 2026 11:54:57 -0700 Subject: [PATCH 2/7] checkstyle fix --- .../nrc/cadc/dali/tables/votable/VOTableReader.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/cadc-dali/src/main/java/ca/nrc/cadc/dali/tables/votable/VOTableReader.java b/cadc-dali/src/main/java/ca/nrc/cadc/dali/tables/votable/VOTableReader.java index a3173cdc..9e0d81aa 100644 --- a/cadc-dali/src/main/java/ca/nrc/cadc/dali/tables/votable/VOTableReader.java +++ b/cadc-dali/src/main/java/ca/nrc/cadc/dali/tables/votable/VOTableReader.java @@ -69,13 +69,11 @@ package ca.nrc.cadc.dali.tables.votable; -import ca.nrc.cadc.dali.tables.TableReader; -import static ca.nrc.cadc.dali.tables.votable.VOTableWriter.SerializationType.BINARY; -import static ca.nrc.cadc.dali.tables.votable.VOTableWriter.SerializationType.BINARY2; - import ca.nrc.cadc.dali.tables.BinaryTableData; import ca.nrc.cadc.dali.tables.ListTableData; import ca.nrc.cadc.dali.tables.TableData; +import ca.nrc.cadc.dali.tables.TableReader; +import ca.nrc.cadc.dali.tables.votable.VOTableWriter.SerializationType; import ca.nrc.cadc.dali.util.Format; import ca.nrc.cadc.dali.util.FormatFactory; import ca.nrc.cadc.util.StringUtil; @@ -343,10 +341,11 @@ protected VOTableDocument readImpl(Reader reader) // TODO: check for href in which case encoding may be irrelevant? final String encoding = streamData.getAttributeValue("encoding", VOTableReader.DEFAULT_STREAM_ENCODING); - if (binaryData.getName().equals(BINARY.name()) || binaryData.getName().equals(BINARY2.name())) { + if (binaryData.getName().equals(SerializationType.BINARY.name()) + || binaryData.getName().equals(SerializationType.BINARY2.name())) { vot.setTableData(new BinaryTableData( new ByteArrayInputStream(streamData.getText().getBytes(StandardCharsets.UTF_8)), - vot.getFields(), encoding, formatFactory, binaryData.getName().equals(BINARY2.name()))); + vot.getFields(), encoding, formatFactory, binaryData.getName().equals(SerializationType.BINARY2.name()))); } else { throw new UnsupportedOperationException("Unsupported type: " + binaryData.getName()); } From d7ed2b1b444497065bab03ecec10274f5deef8d4 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 7 Apr 2026 12:02:07 -0700 Subject: [PATCH 3/7] cadc-dali-parquet: tweak build to work with guava variants --- cadc-dali-parquet/build.gradle | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cadc-dali-parquet/build.gradle b/cadc-dali-parquet/build.gradle index ed3c6f2a..a1ae24bc 100644 --- a/cadc-dali-parquet/build.gradle +++ b/cadc-dali-parquet/build.gradle @@ -35,6 +35,16 @@ dependencies { testImplementation 'xerces:xercesImpl:[2.0,3.0)' } +// work around for google guava variant builds (gradle 6 compat) +import org.gradle.api.attributes.java.TargetJvmEnvironment +configurations.all { + attributes { + // Tell Gradle this environment is a standard JVM, not Android + attribute(TargetJvmEnvironment.TARGET_JVM_ENVIRONMENT_ATTRIBUTE, + objects.named(TargetJvmEnvironment, TargetJvmEnvironment.STANDARD_JVM)) + } +} + // end applications could also add the following exclude // as all these libs are not needed configurations { From 322e0aa528a1ca599e0e2721fc76f0a4b2859da0 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 7 Apr 2026 12:21:39 -0700 Subject: [PATCH 4/7] super ugly workaround for guava dpendency handling --- cadc-dali-parquet/build.gradle | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/cadc-dali-parquet/build.gradle b/cadc-dali-parquet/build.gradle index a1ae24bc..56f88e70 100644 --- a/cadc-dali-parquet/build.gradle +++ b/cadc-dali-parquet/build.gradle @@ -35,13 +35,18 @@ dependencies { testImplementation 'xerces:xercesImpl:[2.0,3.0)' } -// work around for google guava variant builds (gradle 6 compat) -import org.gradle.api.attributes.java.TargetJvmEnvironment +// HACK: needed because of guava variants configurations.all { - attributes { - // Tell Gradle this environment is a standard JVM, not Android - attribute(TargetJvmEnvironment.TARGET_JVM_ENVIRONMENT_ATTRIBUTE, - objects.named(TargetJvmEnvironment, TargetJvmEnvironment.STANDARD_JVM)) + def envAttr = Attribute.of("org.gradle.jvm.environment", String) + + try { + // Try the Gradle 7/8 way (using the actual Class) + def targetJvmClass = Class.forName("org.gradle.api.attributes.java.TargetJvmEnvironment") + def jvmAttr = Attribute.of("org.gradle.jvm.environment", targetJvmClass) + attributes.attribute(jvmAttr, objects.named(targetJvmClass, "standard-jvm")) + } catch (ClassNotFoundException e) { + // Fallback for Gradle 6 (using the String) + attributes.attribute(envAttr, "standard-jvm") } } From 7b49b939f15f62f02788fdbf8d756204f8c49c55 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 7 Apr 2026 12:32:15 -0700 Subject: [PATCH 5/7] cadc-dali-parquet: restrict hadoop version to retain java 11 support --- cadc-dali-parquet/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cadc-dali-parquet/build.gradle b/cadc-dali-parquet/build.gradle index 56f88e70..94bf1b12 100644 --- a/cadc-dali-parquet/build.gradle +++ b/cadc-dali-parquet/build.gradle @@ -24,9 +24,9 @@ dependencies { implementation 'org.opencadc:cadc-util:[1.12.10,)' implementation 'org.opencadc:cadc-dali:[1.2.30,)' - implementation 'org.apache.hadoop:hadoop-common:[3.3.6,)' + implementation 'org.apache.hadoop:hadoop-common:[3.3.6,3.5)' implementation 'org.apache.parquet:parquet-hadoop:[1.13.1,)' - implementation 'org.apache.hadoop:hadoop-mapreduce-client-core:[3.3.6,)' + implementation 'org.apache.hadoop:hadoop-mapreduce-client-core:[3.3.6,3.5)' // CVE-2025-52999 runtimeOnly 'com.fasterxml.jackson.core:jackson-core:[2.15,)' From 19e25e48a5e9f5dc94590520484c997e706c872f Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 7 Apr 2026 12:36:22 -0700 Subject: [PATCH 6/7] checkstyle fix --- .../java/ca/nrc/cadc/dali/tables/parquet/ParquetReader.java | 3 +-- .../java/ca/nrc/cadc/dali/tables/parquet/ParquetWriter.java | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/cadc-dali-parquet/src/main/java/ca/nrc/cadc/dali/tables/parquet/ParquetReader.java b/cadc-dali-parquet/src/main/java/ca/nrc/cadc/dali/tables/parquet/ParquetReader.java index da88d9dd..6d59d816 100644 --- a/cadc-dali-parquet/src/main/java/ca/nrc/cadc/dali/tables/parquet/ParquetReader.java +++ b/cadc-dali-parquet/src/main/java/ca/nrc/cadc/dali/tables/parquet/ParquetReader.java @@ -70,7 +70,6 @@ package ca.nrc.cadc.dali.tables.parquet; import ca.nrc.cadc.dali.tables.TableReader; -import static ca.nrc.cadc.dali.tables.parquet.ParquetWriter.IVOA_VOTABLE_PARQUET_CONTENT_KEY; import ca.nrc.cadc.dali.tables.parquet.io.RandomSeekableInputFile; import ca.nrc.cadc.dali.tables.parquet.readerhelper.ParquetTableData; import ca.nrc.cadc.dali.tables.votable.VOTableDocument; @@ -198,7 +197,7 @@ private VOTableDocument readEmptyVOTable(RandomSeekableInputFile inputFile) thro ParquetMetadata metadata = reader.getFooter(); parquetSchema = metadata.getFileMetaData().getSchema(); - String votable = metadata.getFileMetaData().getKeyValueMetaData().get(IVOA_VOTABLE_PARQUET_CONTENT_KEY); + String votable = metadata.getFileMetaData().getKeyValueMetaData().get(ParquetWriter.IVOA_VOTABLE_PARQUET_CONTENT_KEY); voTableDocument = getVOTableDocument(votable, parquetSchema); votableFields = voTableDocument.getResourceByType("results").getTable().getFields(); diff --git a/cadc-dali-parquet/src/main/java/ca/nrc/cadc/dali/tables/parquet/ParquetWriter.java b/cadc-dali-parquet/src/main/java/ca/nrc/cadc/dali/tables/parquet/ParquetWriter.java index 365f85a5..04ad8125 100644 --- a/cadc-dali-parquet/src/main/java/ca/nrc/cadc/dali/tables/parquet/ParquetWriter.java +++ b/cadc-dali-parquet/src/main/java/ca/nrc/cadc/dali/tables/parquet/ParquetWriter.java @@ -89,7 +89,6 @@ import ca.nrc.cadc.dali.util.URIFormat; import ca.nrc.cadc.dali.util.UUIDFormat; import ca.nrc.cadc.util.HexUtil; - import java.io.BufferedWriter; import java.io.IOException; import java.io.OutputStream; @@ -105,7 +104,6 @@ import java.util.Iterator; import java.util.List; import java.util.Map; - import org.apache.hadoop.conf.Configuration; import org.apache.log4j.Logger; import org.apache.parquet.hadoop.ParquetFileWriter; From 29d84a684d2a1cc41d3abe4957d282bdbef083fb Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 7 Apr 2026 12:46:39 -0700 Subject: [PATCH 7/7] dap: add missing cadc-registry dependency --- dap/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/dap/build.gradle b/dap/build.gradle index 600502d9..3687b0d3 100644 --- a/dap/build.gradle +++ b/dap/build.gradle @@ -29,6 +29,7 @@ dependencies { implementation 'org.opencadc:cadc-util:[1.6.1,)' implementation 'org.opencadc:cadc-rest:[1.4.5,)' + implementation 'org.opencadc:cadc-registry:[1.9,)' implementation 'org.opencadc:cadc-cdp:[1.2.3,)' implementation 'org.opencadc:cadc-uws:[1.0.5,)' implementation 'org.opencadc:cadc-uws-server:[1.2.4,)'